this repo has no description
Log exit reasons
+50
-21
+5
-2
lib/servers/begin_buffer.ex
+5
-2
lib/servers/begin_buffer.ex
···
1
1
defmodule Hobbes.Servers.BeginBuffer do
2
2
use GenServer
3
-
alias Trinity.{SimProcess, SimServer}
3
+
alias Trinity.{SimProcess, SimServer, SimLogger}
4
+
require SimLogger
4
5
5
6
import ExUnit.Assertions, only: [assert: 1]
6
7
···
79
80
80
81
# If a TLog is locked in this generation there is an ongoing recovery,
81
82
# so we may as well just die instead of trying to handle this case
82
-
true -> exit(:shutdown)
83
+
true ->
84
+
SimLogger.debug "BeginBuffer (id=#{state.id}, generation=#{state.cluster.generation}): TLog reported locked in reply, exiting"
85
+
exit(:shutdown)
83
86
end
84
87
end
85
88
+16
-7
lib/servers/commit_buffer.ex
+16
-7
lib/servers/commit_buffer.ex
···
207
207
208
208
{commit_version, prev_commit_version} =
209
209
case Sequencer.get_commit_version(seq_pid) do
210
-
{:ok, {_cv, _pcv} = versions} -> versions
211
-
{:error, :timeout} -> exit(:shutdown)
210
+
{:ok, {_cv, _pcv} = versions} ->
211
+
versions
212
+
{:error, :timeout} ->
213
+
SimLogger.debug "CommitBuffer (id=#{state.id}, generation=#{state.cluster.generation})): failed to get commit version, exiting"
214
+
exit(:shutdown)
212
215
end
213
216
214
217
transactions_reversed = state.buffer
···
246
249
[%Server{pid: resolver_pid}] = get_servers(state.cluster, Hobbes.Servers.Resolver)
247
250
{txn_results_reversed, meta_log} =
248
251
case Resolver.resolve_batch(resolver_pid, batch) do
249
-
{:ok, result} -> result
250
-
{:error, :timeout} -> exit(:shutdown)
252
+
{:ok, result} ->
253
+
result
254
+
{:error, :timeout} ->
255
+
SimLogger.debug "CommitBuffer (id=#{state.id}, generation=#{state.cluster.generation})): failed to resolve batch, exiting"
256
+
exit(:shutdown)
251
257
end
252
258
253
259
# Apply meta mutations received, including our own from this batch
···
307
313
case TLog.write_batch_receive(req_id) do
308
314
:ok -> :noop
309
315
# If a commit fails, we trigger recovery (or are already recovering)
310
-
{:error, :tlog_locked} -> exit(:shutdown)
311
-
{:error, :timeout} -> exit(:shutdown)
316
+
{:error, err} when err in [:tlog_locked, :timeout] ->
317
+
SimLogger.debug "CommitBuffer (id=#{state.id}, generation=#{state.cluster.generation})): failed to write batch to TLog (error=#{err}), exiting"
318
+
exit(:shutdown)
312
319
end
313
320
end)
314
321
315
322
# Once all tlogs have replied (made durable), notify sequencer this version is committed
316
323
case Sequencer.notify_committed(seq_pid, commit_version) do
317
324
:ok -> :noop
318
-
{:error, :timeout} -> exit(:shutdown)
325
+
{:error, :timeout} ->
326
+
SimLogger.debug "CommitBuffer (id=#{state.id}, generation=#{state.cluster.generation})): failed to notify sequencer of commit, exiting"
327
+
exit(:shutdown)
319
328
end
320
329
321
330
# Reply to clients
+2
-1
lib/servers/coordinator.ex
+2
-1
lib/servers/coordinator.ex
···
371
371
SimProcess.send_after(self(), :tick, @tick_interval_ms)
372
372
SimProcess.send_after(self(), :work, @work_interval_ms)
373
373
374
-
SimLogger.debug "Coordinator (id=#{state.id}) initialized"
374
+
debug_info = Map.take(state, [:view_number, :last_normal_view_number, :op_number, :commit_number, :version])
375
+
SimLogger.debug "Coordinator (id=#{state.id}) initialized: #{inspect(debug_info)}"
375
376
{:ok, state}
376
377
end
377
378
+18
-7
lib/servers/manager.ex
+18
-7
lib/servers/manager.ex
···
116
116
# If a transaction process in this generation has failed, shutdown immediately
117
117
# so that a new Manager can be recruited
118
118
if failed? do
119
+
SimLogger.debug "Manager (generation=#{state.generation}): A supervisor reported failure, exiting"
119
120
exit(:shutdown)
120
121
end
121
122
···
138
139
old_generations: generation_pairs,
139
140
} =
140
141
case Coordinator.inc_generation(state.primary_coordinator) do
141
-
{:ok, result} -> result
142
-
{:error, :no_config} -> exit(:shutdown)
143
-
{:error, :timeout} -> exit(:shutdown)
142
+
{:ok, result} ->
143
+
result
144
+
145
+
{:error, err} when err in [:no_config, :timeout] ->
146
+
SimLogger.debug "Manager failed to inc_generation (error=#{inspect(err)}), exiting"
147
+
exit(:shutdown)
144
148
end
145
149
146
150
assert is_integer(generation)
···
202
206
203
207
if elapsed_us > @supervisor_timeout_us do
204
208
# If a supervisor fails to ping for too long we trigger a recovery
209
+
# TODO: log TLog id
210
+
SimLogger.debug "Manager (generation=#{state.generation}): TLog failed to ping for #{elapsed_us}us, exiting"
205
211
exit(:shutdown)
206
212
end
207
213
end)
···
581
587
end)
582
588
|> Enum.map(fn req_id ->
583
589
case TLog.peek_all_receive(req_id) do
584
-
{:ok, batches} -> batches
585
-
{:error, :timeout} -> exit(:shutdown)
590
+
{:ok, batches} ->
591
+
batches
592
+
{:error, :timeout} ->
593
+
SimLogger.debug "Manager (generation=#{state.generation}): TLog peek timed out during copy, exiting"
594
+
exit(:shutdown)
586
595
end
587
596
end)
588
597
|> Enum.concat()
···
780
789
{:ok, server_pid} ->
781
790
server_pid
782
791
{:error, err} ->
783
-
SimLogger.debug("Manager failed to recruit server (generation: #{generation}, module: #{module}, error: #{err}) - exiting")
792
+
SimLogger.debug("Manager (generation=#{generation}): failed to recruit server (module: #{module}, error: #{err}), exiting")
784
793
exit(:shutdown)
785
794
end
786
795
···
862
871
|> then(fn {[], acc} -> acc end)
863
872
|> Map.new()
864
873
865
-
{:error, _err} -> exit(:shutdown)
874
+
{:error, _err} ->
875
+
SimLogger.debug "Manager (generation=#{state.generation}): failed to allocate ids, exiting"
876
+
exit(:shutdown)
866
877
end
867
878
end
868
879
+8
-4
lib/servers/storage.ex
+8
-4
lib/servers/storage.ex
···
1
1
defmodule Hobbes.Servers.Storage do
2
2
use GenServer
3
-
alias Trinity.{Sim, SimProcess, SimServer}
4
-
require Logger
3
+
alias Trinity.{Sim, SimProcess, SimServer, SimLogger}
4
+
require SimLogger
5
5
6
6
import ExUnit.Assertions, only: [assert: 1]
7
7
···
382
382
new_gen = cluster.tlog_generations |> Enum.map(&(&1.generation)) |> Enum.max(&>=/2, fn -> 1 end)
383
383
has_recovered? = new_gen > old_gen
384
384
# If there has been a recovery (a new generation > 1 appears in cluster.tlog_generations),
385
-
# we purge possibly-uncommitted mutations from the in-memory KV by simply restarting
386
-
if has_recovered?, do: exit(:shutdown)
385
+
# we undo any mutations that may be excluded from the new generation by restarting the server
386
+
# (these mutations are guaranteed not to be persisted via XKS.set_max_persist_version)
387
+
if has_recovered? do
388
+
SimLogger.debug "Storage (id=#{state.id}, generation=#{state.cluster.generation}): received a new TLog generation (old=#{old_gen}, new=#{new_gen}), exiting"
389
+
exit(:shutdown)
390
+
end
387
391
388
392
state = %{state | cluster: cluster}
389
393