Deployment and lifecycle management for Nix
fix: stabilize flaky channel tests with async reconciliation and task draining
- Run reconcile_deployments in a background task instead of blocking the
channel process, preventing assert_reply timeouts in slow environments
- Drain TaskSupervisor children in channel_case on_exit to prevent
sandbox teardown while tasks are still running
- Increase assert_reply timeouts from 100ms default to 1000ms
- Use wait_until_succeeds for garden deploy RPC in e2e test to handle
subscription sync race
sow-125
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+39
-22
+5
-1
apps/sower/lib/sower_web/garden_channel.ex
+5
-1
apps/sower/lib/sower_web/garden_channel.ex
···
222
222
:reconcile_deployments,
223
223
%Phoenix.Socket{assigns: %{garden: garden}} = socket
224
224
) do
225
-
Orchestration.Deployment.reconcile_deployments_on_connect(garden)
225
+
Task.Supervisor.start_child(Sower.TaskSupervisor, fn ->
226
+
Sower.Repo.put_org_id(garden.org_id)
227
+
Orchestration.Deployment.reconcile_deployments_on_connect(garden)
228
+
end)
229
+
226
230
{:noreply, socket}
227
231
end
228
232
+20
-20
apps/sower/test/sower_web/channels/garden_channel_handle_in_test.exs
+20
-20
apps/sower/test/sower_web/channels/garden_channel_handle_in_test.exs
···
6
6
%{socket: socket} = connect_and_join_garden()
7
7
8
8
ref = push(socket, "ping", %{})
9
-
assert_reply ref, :ok, :pong
9
+
assert_reply ref, :ok, :pong, 1_000
10
10
end
11
11
end
12
12
···
21
21
"name" => garden.name
22
22
})
23
23
24
-
assert_reply ref, :ok, reply
24
+
assert_reply ref, :ok, reply, 1_000
25
25
assert reply.sid == garden.sid
26
26
end
27
27
···
65
65
"name" => garden.name
66
66
})
67
67
68
-
assert_reply ref, :ok, reply
68
+
assert_reply ref, :ok, reply, 1_000
69
69
assert reply.sid == garden.sid
70
70
end
71
71
···
79
79
"name" => garden.name
80
80
})
81
81
82
-
assert_reply ref, :ok, reply
82
+
assert_reply ref, :ok, reply, 1_000
83
83
assert reply.sid == garden.sid
84
84
end
85
85
end
···
103
103
"subscription_sids" => [subscription.sid]
104
104
})
105
105
106
-
assert_reply ref, :ok, %{request_id: request_id}
106
+
assert_reply ref, :ok, %{request_id: request_id}, 1_000
107
107
assert is_binary(request_id)
108
108
109
109
# Wait for the async deployment task to complete before test exits
···
120
120
"subscription_sids" => ["nonexistent_sid"]
121
121
})
122
122
123
-
assert_reply ref, :error, :subscription_not_found
123
+
assert_reply ref, :error, :subscription_not_found, 1_000
124
124
end
125
125
end
126
126
···
150
150
"status" => "acknowledged"
151
151
})
152
152
153
-
assert_reply ref, :ok, reply
153
+
assert_reply ref, :ok, reply, 1_000
154
154
assert reply.state == :acknowledged
155
155
end
156
156
···
164
164
"status" => "acknowledged"
165
165
})
166
166
167
-
assert_reply ref, :error, :deployment_not_found
167
+
assert_reply ref, :error, :deployment_not_found, 1_000
168
168
end
169
169
end
170
170
···
196
196
"deployed_at" => DateTime.utc_now() |> DateTime.to_iso8601()
197
197
})
198
198
199
-
assert_reply ref, :ok, reply
199
+
assert_reply ref, :ok, reply, 1_000
200
200
assert reply.state == :completed
201
201
assert reply.result == :success
202
202
end
···
212
212
"result" => "success"
213
213
})
214
214
215
-
assert_reply ref, :error, :deployment_not_found
215
+
assert_reply ref, :error, :deployment_not_found, 1_000
216
216
end
217
217
end
218
218
···
243
243
"status" => "downloading"
244
244
})
245
245
246
-
assert_reply ref, :ok, %{}
246
+
assert_reply ref, :ok, %{}, 1_000
247
247
end
248
248
249
249
@tag :capture_log
···
257
257
"status" => "downloading"
258
258
})
259
259
260
-
assert_reply ref, :error, :deployment_not_found
260
+
assert_reply ref, :error, :deployment_not_found, 1_000
261
261
end
262
262
263
263
@tag :capture_log
···
288
288
"status" => "downloading"
289
289
})
290
290
291
-
assert_reply ref, :error, :seed_not_in_deployment
291
+
assert_reply ref, :error, :seed_not_in_deployment, 1_000
292
292
end
293
293
end
294
294
···
320
320
"log" => "deployment completed successfully"
321
321
})
322
322
323
-
assert_reply ref, :ok, %{}
323
+
assert_reply ref, :ok, %{}, 1_000
324
324
325
325
[seed_deployment] =
326
326
Sower.Repo.preload(deployment, :seed_deployments, force: true).seed_deployments
···
355
355
"log" => "partial output"
356
356
})
357
357
358
-
assert_reply ref, :ok, %{}
358
+
assert_reply ref, :ok, %{}, 1_000
359
359
360
360
[seed_deployment] =
361
361
Sower.Repo.preload(deployment, :seed_deployments, force: true).seed_deployments
···
375
375
"result" => "success"
376
376
})
377
377
378
-
assert_reply ref, :error, :deployment_not_found
378
+
assert_reply ref, :error, :deployment_not_found, 1_000
379
379
end
380
380
end
381
381
···
391
391
]
392
392
})
393
393
394
-
assert_reply ref, :ok, %{subscriptions: subscriptions}
394
+
assert_reply ref, :ok, %{subscriptions: subscriptions}, 1_000
395
395
assert length(subscriptions) == 2
396
396
397
397
names = Enum.map(subscriptions, & &1.seed_name) |> Enum.sort()
···
414
414
]
415
415
})
416
416
417
-
assert_reply ref, :ok, %{subscriptions: subscriptions}
417
+
assert_reply ref, :ok, %{subscriptions: subscriptions}, 1_000
418
418
assert length(subscriptions) == 1
419
419
assert hd(subscriptions).seed_name == "to-keep"
420
420
···
448
448
]
449
449
})
450
450
451
-
assert_reply ref, :ok, :ok
451
+
assert_reply ref, :ok, :ok, 1_000
452
452
end
453
453
454
454
test "handles empty profiles list" do
···
459
459
"profiles" => []
460
460
})
461
461
462
-
assert_reply ref, :ok, :ok
462
+
assert_reply ref, :ok, :ok, 1_000
463
463
end
464
464
end
465
465
end
+13
apps/sower/test/support/channel_case.ex
+13
apps/sower/test/support/channel_case.ex
···
15
15
16
16
setup tags do
17
17
Sower.DataCase.setup_sandbox(tags)
18
+
19
+
on_exit(fn ->
20
+
for pid <- Task.Supervisor.children(Sower.TaskSupervisor) do
21
+
ref = Process.monitor(pid)
22
+
23
+
receive do
24
+
{:DOWN, ^ref, :process, ^pid, _} -> :ok
25
+
after
26
+
5_000 -> :ok
27
+
end
28
+
end
29
+
end)
30
+
18
31
:ok
19
32
end
20
33
+1
-1
nix/tests/e2e.nix
+1
-1
nix/tests/e2e.nix
···
212
212
server.succeed("sower seed upgrade --name server --type nixos --debug")
213
213
214
214
with subtest("nixos garden deployment"):
215
-
server.succeed('sower-garden rpc "Garden.Admin.deploy(\\\"nixos\\\")"')
215
+
server.wait_until_succeeds('sower-garden rpc "Garden.Admin.deploy(\\\"nixos\\\")"', timeout=15)
216
216
server.wait_until_succeeds(
217
217
"journalctl --no-pager -u sower-garden"
218
218
" --grep='Completed.activation'",