Deployment and lifecycle management for Nix
test: catch some more failures and silence logs
+95
-63
+63
-52
apps/sower/lib/sower/orchestration/deployment.ex
+63
-52
apps/sower/lib/sower/orchestration/deployment.ex
···
313
313
end)
314
314
315
315
# 4. Deploy fresh for all overdue subscriptions
316
-
Enum.each(overdue, fn sub ->
317
-
deploy_subscription(sub)
316
+
overdue_tasks =
317
+
Enum.map(overdue, fn sub ->
318
+
{:ok, _request_id, pid} = deploy_subscription(sub)
319
+
pid
320
+
end)
321
+
322
+
Enum.each(overdue_tasks, fn pid ->
323
+
ref = Process.monitor(pid)
324
+
325
+
receive do
326
+
{:DOWN, ^ref, :process, ^pid, _reason} -> :ok
327
+
end
318
328
end)
319
329
320
330
if to_replay != [] or overdue != [] or to_cancel != [] do
···
361
371
362
372
%Garden{} = garden ->
363
373
request_id = SowerClient.Sid.generate("req")
364
-
{:ok, request_id} = process_deployment(request_id, [subscription], garden, opts)
365
-
{:ok, request_id}
374
+
{:ok, request_id, pid} = process_deployment(request_id, [subscription], garden, opts)
375
+
{:ok, request_id, pid}
366
376
end
367
377
end
368
378
···
381
391
garden
382
392
) do
383
393
with {:ok, subscriptions} <- validate_deployment_request(request, garden.id),
384
-
{:ok, request_id} <-
394
+
{:ok, request_id, pid} <-
385
395
process_deployment(request.request_id, subscriptions, garden, force: request.force) do
386
-
{:ok, request_id}
396
+
{:ok, request_id, pid}
387
397
end
388
398
end
389
399
390
400
def process_deployment(request_id, subscriptions, %Garden{} = garden, opts \\ []) do
391
-
Task.Supervisor.start_child(Sower.TaskSupervisor, fn ->
392
-
Repo.put_org_id(garden.org_id)
401
+
{:ok, pid} =
402
+
Task.Supervisor.start_child(Sower.TaskSupervisor, fn ->
403
+
Repo.put_org_id(garden.org_id)
393
404
394
-
Logger.info(
395
-
msg: "Deployment processing started",
396
-
request_id: request_id,
397
-
garden_id: garden.id
398
-
)
405
+
Logger.info(
406
+
msg: "Deployment processing started",
407
+
request_id: request_id,
408
+
garden_id: garden.id
409
+
)
399
410
400
-
case do_deployment(request_id, subscriptions, opts) do
401
-
{:ok, deployment} ->
402
-
Logger.info(
403
-
msg: "Deployment broadcast successful",
404
-
request_id: request_id,
405
-
deployment_sid: deployment.sid,
406
-
skipped: deployment.skipped
407
-
)
411
+
case do_deployment(request_id, subscriptions, opts) do
412
+
{:ok, deployment} ->
413
+
Logger.info(
414
+
msg: "Deployment broadcast successful",
415
+
request_id: request_id,
416
+
deployment_sid: deployment.sid,
417
+
skipped: deployment.skipped
418
+
)
408
419
409
-
SowerWeb.Endpoint.broadcast(
410
-
"garden:#{garden.sid}",
411
-
"deployment",
412
-
Map.from_struct(deployment)
413
-
)
420
+
SowerWeb.Endpoint.broadcast(
421
+
"garden:#{garden.sid}",
422
+
"deployment",
423
+
Map.from_struct(deployment)
424
+
)
414
425
415
-
# Backward compatibility: 0.7.0 gardens join "agent:*"
416
-
SowerWeb.Endpoint.broadcast(
417
-
"agent:#{garden.sid}",
418
-
"deployment",
419
-
Map.from_struct(deployment)
420
-
)
426
+
# Backward compatibility: 0.7.0 gardens join "agent:*"
427
+
SowerWeb.Endpoint.broadcast(
428
+
"agent:#{garden.sid}",
429
+
"deployment",
430
+
Map.from_struct(deployment)
431
+
)
421
432
422
-
{:error, reason} ->
423
-
Logger.error(
424
-
msg: "Deployment processing failed",
425
-
request_id: request_id,
426
-
reason: to_string(reason)
427
-
)
433
+
{:error, reason} ->
434
+
Logger.error(
435
+
msg: "Deployment processing failed",
436
+
request_id: request_id,
437
+
reason: to_string(reason)
438
+
)
428
439
429
-
SowerWeb.Endpoint.broadcast(
430
-
"garden:#{garden.sid}",
431
-
"deployment:error",
432
-
%{request_id: request_id, reason: to_string(reason)}
433
-
)
440
+
SowerWeb.Endpoint.broadcast(
441
+
"garden:#{garden.sid}",
442
+
"deployment:error",
443
+
%{request_id: request_id, reason: to_string(reason)}
444
+
)
434
445
435
-
# Backward compatibility: 0.7.0 gardens join "agent:*"
436
-
SowerWeb.Endpoint.broadcast(
437
-
"agent:#{garden.sid}",
438
-
"deployment:error",
439
-
%{request_id: request_id, reason: to_string(reason)}
440
-
)
441
-
end
442
-
end)
446
+
# Backward compatibility: 0.7.0 gardens join "agent:*"
447
+
SowerWeb.Endpoint.broadcast(
448
+
"agent:#{garden.sid}",
449
+
"deployment:error",
450
+
%{request_id: request_id, reason: to_string(reason)}
451
+
)
452
+
end
453
+
end)
443
454
444
-
{:ok, request_id}
455
+
{:ok, request_id, pid}
445
456
end
446
457
447
458
def record_deployment_status(%SowerClient.Orchestration.DeploymentStatus{} = status) do
+1
-1
apps/sower/lib/sower/orchestration/workflows/realtime_deploy.ex
+1
-1
apps/sower/lib/sower/orchestration/workflows/realtime_deploy.ex
+1
-1
apps/sower/lib/sower_web/garden_channel.ex
+1
-1
apps/sower/lib/sower_web/garden_channel.ex
+1
-1
apps/sower/lib/sower_web/live/garden_live/show.ex
+1
-1
apps/sower/lib/sower_web/live/garden_live/show.ex
+1
-1
apps/sower/lib/sower_web/live/subscription_live/show.ex
+1
-1
apps/sower/lib/sower_web/live/subscription_live/show.ex
+5
-1
apps/sower/mix.exs
+5
-1
apps/sower/mix.exs
···
83
83
defp aliases do
84
84
[
85
85
setup: ["deps.get", "ecto.setup", "assets.setup", "assets.build"],
86
-
"ecto.setup": ["ecto.create", "ecto.migrate", "run #{Path.expand("priv/repo/seeds.exs", __DIR__)}"],
86
+
"ecto.setup": [
87
+
"ecto.create",
88
+
"ecto.migrate",
89
+
"run #{Path.expand("priv/repo/seeds.exs", __DIR__)}"
90
+
],
87
91
"ecto.reset": ["ecto.drop", "ecto.setup"],
88
92
test: ["ecto.create --quiet", "ecto.migrate --quiet", "test"],
89
93
"assets.setup": ["tailwind.install --if-missing", "esbuild.install --if-missing"],
+2
apps/sower/test/sower/accounts/user_test.exs
+2
apps/sower/test/sower/accounts/user_test.exs
···
24
24
end
25
25
26
26
describe "new/1" do
27
+
@tag :capture_log
27
28
test "validates email uniqueness" do
28
29
%{email: email, org_id: org_id} = user_fixture()
29
30
···
55
56
%{user: user_fixture()}
56
57
end
57
58
59
+
@tag :capture_log
58
60
test "generates a token", %{user: user, organization: org} do
59
61
token = User.generate_session_token(user)
60
62
assert user_token = Repo.get_by(UserToken, [token: token], skip_org_id: true)
-3
apps/sower/test/sower/orchestration/deployment_sequence_test.exs
-3
apps/sower/test/sower/orchestration/deployment_sequence_test.exs
+14
-3
apps/sower/test/sower/orchestration_test.exs
+14
-3
apps/sower/test/sower/orchestration_test.exs
···
505
505
assert asp.is_current == true
506
506
end
507
507
508
+
@tag :capture_log
508
509
test "upsert_from_report/4 updates existing profile on conflict" do
509
510
garden = garden_fixture()
510
511
seed = seed_fixture()
···
532
533
assert asp2.is_current == true
533
534
end
534
535
536
+
@tag :capture_log
535
537
test "unique constraint on garden_id and seed_id" do
536
538
garden = garden_fixture()
537
539
seed = seed_fixture()
···
922
924
force: false
923
925
})
924
926
925
-
assert {:ok, request_id} = Orchestration.handle_deployment_request(request, garden)
927
+
assert {:ok, request_id, pid} = Orchestration.handle_deployment_request(request, garden)
926
928
assert is_binary(request_id)
929
+
930
+
ref = Process.monitor(pid)
931
+
assert_receive {:DOWN, ^ref, :process, ^pid, :normal}, 5000
927
932
end
928
933
929
934
test "returns error for deployment request with unauthorized subscription", %{
···
966
971
967
972
request_id = "dr_test_#{System.unique_integer([:positive])}"
968
973
969
-
assert {:ok, ^request_id} =
974
+
assert {:ok, ^request_id, pid} =
970
975
Orchestration.process_deployment(request_id, [subscription], garden)
976
+
977
+
ref = Process.monitor(pid)
978
+
assert_receive {:DOWN, ^ref, :process, ^pid, :normal}, 5000
971
979
end
972
980
973
981
@tag :capture_log
···
984
992
985
993
request_id = "dr_test_error_#{System.unique_integer([:positive])}"
986
994
987
-
assert {:ok, ^request_id} =
995
+
assert {:ok, ^request_id, pid} =
988
996
Orchestration.process_deployment(request_id, [subscription], garden)
997
+
998
+
ref = Process.monitor(pid)
999
+
assert_receive {:DOWN, ^ref, :process, ^pid, :normal}, 5000
989
1000
end
990
1001
end
991
1002
+1
apps/sower/test/sower/seed_test.exs
+1
apps/sower/test/sower/seed_test.exs
+6
apps/sower/test/sower_web/channels/garden_channel_handle_in_test.exs
+6
apps/sower/test/sower_web/channels/garden_channel_handle_in_test.exs
···
107
107
assert is_binary(request_id)
108
108
end
109
109
110
+
@tag :capture_log
110
111
test "returns error for unknown subscription" do
111
112
%{socket: socket} = connect_and_join_garden()
112
113
···
150
151
assert reply.state == :acknowledged
151
152
end
152
153
154
+
@tag :capture_log
153
155
test "returns error for unknown deployment" do
154
156
%{socket: socket} = connect_and_join_garden()
155
157
···
196
198
assert reply.result == :success
197
199
end
198
200
201
+
@tag :capture_log
199
202
test "returns error for unknown deployment" do
200
203
%{socket: socket} = connect_and_join_garden()
201
204
···
240
243
assert_reply ref, :ok, %{}
241
244
end
242
245
246
+
@tag :capture_log
243
247
test "returns error for unknown deployment" do
244
248
%{socket: socket} = connect_and_join_garden()
245
249
···
253
257
assert_reply ref, :error, :deployment_not_found
254
258
end
255
259
260
+
@tag :capture_log
256
261
test "returns error for seed not in deployment" do
257
262
%{socket: socket, garden: garden} = connect_and_join_garden()
258
263
···
356
361
assert seed_deployment.log == "partial output"
357
362
end
358
363
364
+
@tag :capture_log
359
365
test "returns error for unknown deployment" do
360
366
%{socket: socket} = connect_and_join_garden()
361
367