Deployment and lifecycle management for Nix
agent: ensure s3 log includes missing sower-activator
+180
-43
+71
-43
apps/sower_agent/lib/sower_agent/deployer.ex
+71
-43
apps/sower_agent/lib/sower_agent/deployer.ex
···
47
47
end
48
48
49
49
def upgrade(%Deployment{} = deployment) do
50
-
deployment.seed_deployments
51
-
|> async_stream(fn %{seed: seed} = seed_deploy ->
50
+
upgrade(deployment, [])
51
+
end
52
+
53
+
def upgrade(%Deployment{} = deployment, opts) do
54
+
async_stream_fun = Keyword.get(opts, :async_stream_fun, &async_stream/2)
55
+
realize_seed_fun = Keyword.get(opts, :realize_seed_fun, &realize_seed/1)
56
+
57
+
get_deployment_profile_fun =
58
+
Keyword.get(opts, :get_deployment_profile_fun, &get_deployment_profile/1)
59
+
60
+
activate_seed_fun = Keyword.get(opts, :activate_seed_fun, &SowerAgent.Seed.activate/2)
61
+
write_log_fun = Keyword.get(opts, :write_log_fun, &maybe_write_log/3)
62
+
63
+
async_stream_fun.(deployment.seed_deployments, fn %{seed: seed} = seed_deploy ->
52
64
Logger.debug(
53
65
msg: "Realizing seed",
54
66
name: seed.name,
···
57
69
artifact: seed.artifact
58
70
)
59
71
60
-
case System.cmd("nix-store", ["--realize", seed.artifact],
61
-
stderr_to_stdout: true,
62
-
into: [],
63
-
lines: 1024
64
-
) do
65
-
{_output, 0} ->
66
-
Logger.info(
67
-
msg: "Successfully realized seed",
68
-
name: seed.name,
69
-
seed_sid: seed.sid,
70
-
seed_type: seed.seed_type,
71
-
artifact: seed.artifact
72
-
)
73
-
74
-
{:ok, seed_deploy}
75
-
76
-
{output, exit_code} ->
77
-
output =
78
-
Enum.filter(output, fn line ->
79
-
line not in [
80
-
"warning: you did not specify '--add-root'; the result might be removed by the garbage collector"
81
-
]
82
-
end)
83
-
84
-
Logger.error(
85
-
msg: "Failed to realize seed",
86
-
name: seed.name,
87
-
seed_sid: seed.sid,
88
-
seed_type: seed.seed_type,
89
-
artifact: seed.artifact,
90
-
exit_code: exit_code,
91
-
output: output
92
-
)
93
-
94
-
{:error, :failed_to_realize, seed_deploy}
95
-
end
72
+
realize_seed_fun.(seed_deploy)
96
73
end)
97
-
|> async_stream(fn
74
+
|> async_stream_fun.(fn
98
75
{:ok, {:ok, %SeedDeployment{seed: seed} = seed_deploy}} ->
99
-
profile = get_deployment_profile(seed_deploy.subscription_sid)
76
+
profile = get_deployment_profile_fun.(seed_deploy.subscription_sid)
100
77
101
78
Logger.info(
102
79
msg: "Activating seed",
···
108
85
activation_args: get_in(profile.activation_args)
109
86
)
110
87
111
-
result = SowerAgent.Seed.activate(seed, profile)
88
+
result = activate_seed_fun.(seed, profile)
112
89
113
90
case result do
114
91
{:ok, output} ->
···
118
95
seed_sid: seed.sid
119
96
)
120
97
121
-
maybe_write_log(deployment, seed, output)
98
+
write_log_fun.(deployment, seed, output)
122
99
123
100
{:error, _code, output} ->
124
101
Logger.error(
···
127
104
seed_sid: seed.sid
128
105
)
129
106
130
-
maybe_write_log(deployment, seed, output)
107
+
write_log_fun.(deployment, seed, output)
108
+
109
+
{:error, reason} when reason in [:activator_unavailable, :cmd_not_found] ->
110
+
Logger.error(
111
+
msg: "Missing activator during deployment activation",
112
+
deployment_sid: deployment.sid,
113
+
seed_sid: seed.sid,
114
+
reason: inspect(reason)
115
+
)
116
+
117
+
write_log_fun.(deployment, seed, [
118
+
"FATAL: missing activator executable sower-activator; deployment cannot continue"
119
+
])
131
120
132
121
{:error, _reason} ->
133
122
:ok
···
142
131
error
143
132
end)
144
133
|> Enum.to_list()
134
+
end
135
+
136
+
defp realize_seed(%SeedDeployment{seed: seed} = seed_deploy) do
137
+
case System.cmd("nix-store", ["--realize", seed.artifact],
138
+
stderr_to_stdout: true,
139
+
into: [],
140
+
lines: 1024
141
+
) do
142
+
{_output, 0} ->
143
+
Logger.info(
144
+
msg: "Successfully realized seed",
145
+
name: seed.name,
146
+
seed_sid: seed.sid,
147
+
seed_type: seed.seed_type,
148
+
artifact: seed.artifact
149
+
)
150
+
151
+
{:ok, seed_deploy}
152
+
153
+
{output, exit_code} ->
154
+
output =
155
+
Enum.filter(output, fn line ->
156
+
line not in [
157
+
"warning: you did not specify '--add-root'; the result might be removed by the garbage collector"
158
+
]
159
+
end)
160
+
161
+
Logger.error(
162
+
msg: "Failed to realize seed",
163
+
name: seed.name,
164
+
seed_sid: seed.sid,
165
+
seed_type: seed.seed_type,
166
+
artifact: seed.artifact,
167
+
exit_code: exit_code,
168
+
output: output
169
+
)
170
+
171
+
{:error, :failed_to_realize, seed_deploy}
172
+
end
145
173
end
146
174
147
175
def async_stream(enumerable, func) do
+109
apps/sower_agent/test/sower_agent/deployer_test.exs
+109
apps/sower_agent/test/sower_agent/deployer_test.exs
···
2
2
use ExUnit.Case, async: true
3
3
4
4
import ExUnit.CaptureLog
5
+
require Logger
5
6
6
7
alias SowerAgent.Deployer
7
8
alias SowerClient.Orchestration.Deployment
···
262
263
end
263
264
end
264
265
266
+
describe "upgrade/2" do
267
+
test "writes a fatal deployment log line when activator is unavailable" do
268
+
deployment = %Deployment{
269
+
sid: "dep_1",
270
+
seed_deployments: [seed_deploy_with_identity("seed_1")]
271
+
}
272
+
273
+
logs =
274
+
capture_log(fn ->
275
+
assert [
276
+
{:ok, {:error, :activator_unavailable}}
277
+
] =
278
+
Deployer.upgrade(deployment,
279
+
async_stream_fun: fn enumerable, func ->
280
+
Enum.map(enumerable, fn item -> {:ok, func.(item)} end)
281
+
end,
282
+
realize_seed_fun: fn seed_deploy -> {:ok, seed_deploy} end,
283
+
get_deployment_profile_fun: fn _ -> %DeploymentProfile{} end,
284
+
activate_seed_fun: fn _seed, _profile -> {:error, :activator_unavailable} end,
285
+
write_log_fun: fn _deployment, _seed, output_lines ->
286
+
Logger.error(
287
+
msg: "Deployment log payload",
288
+
output: Enum.join(output_lines, "\n")
289
+
)
290
+
end
291
+
)
292
+
end)
293
+
294
+
assert logs =~ "Missing activator during deployment activation"
295
+
296
+
assert logs =~
297
+
"FATAL: missing activator executable sower-activator; deployment cannot continue"
298
+
end
299
+
300
+
test "keeps successful deployment logging behavior unchanged" do
301
+
deployment = %Deployment{
302
+
sid: "dep_2",
303
+
seed_deployments: [seed_deploy_with_identity("seed_2")]
304
+
}
305
+
306
+
capture_log(fn ->
307
+
assert [
308
+
{:ok, {:ok, ["activation complete"]}}
309
+
] =
310
+
Deployer.upgrade(deployment,
311
+
async_stream_fun: fn enumerable, func ->
312
+
Enum.map(enumerable, fn item -> {:ok, func.(item)} end)
313
+
end,
314
+
realize_seed_fun: fn seed_deploy -> {:ok, seed_deploy} end,
315
+
get_deployment_profile_fun: fn _ -> %DeploymentProfile{} end,
316
+
activate_seed_fun: fn _seed, _profile -> {:ok, ["activation complete"]} end,
317
+
write_log_fun: fn _deployment, _seed, output_lines ->
318
+
Logger.info(
319
+
msg: "Deployment log payload",
320
+
output: Enum.join(output_lines, "\n")
321
+
)
322
+
end
323
+
)
324
+
end)
325
+
end
326
+
327
+
test "writes a fatal deployment log line when activator executable is missing" do
328
+
deployment = %Deployment{
329
+
sid: "dep_3",
330
+
seed_deployments: [seed_deploy_with_identity("seed_3")]
331
+
}
332
+
333
+
logs =
334
+
capture_log(fn ->
335
+
assert [
336
+
{:ok, {:error, :cmd_not_found}}
337
+
] =
338
+
Deployer.upgrade(deployment,
339
+
async_stream_fun: fn enumerable, func ->
340
+
Enum.map(enumerable, fn item -> {:ok, func.(item)} end)
341
+
end,
342
+
realize_seed_fun: fn seed_deploy -> {:ok, seed_deploy} end,
343
+
get_deployment_profile_fun: fn _ -> %DeploymentProfile{} end,
344
+
activate_seed_fun: fn _seed, _profile -> {:error, :cmd_not_found} end,
345
+
write_log_fun: fn _deployment, _seed, output_lines ->
346
+
Logger.error(
347
+
msg: "Deployment log payload",
348
+
output: Enum.join(output_lines, "\n")
349
+
)
350
+
end
351
+
)
352
+
end)
353
+
354
+
assert logs =~ "Missing activator during deployment activation"
355
+
assert logs =~ "cmd_not_found"
356
+
357
+
assert logs =~
358
+
"FATAL: missing activator executable sower-activator; deployment cannot continue"
359
+
end
360
+
end
361
+
265
362
defp seed_deploy(subscription_sid, seed_type \\ "nixos") do
266
363
%SeedDeployment{
267
364
subscription_sid: subscription_sid,
268
365
seed: %Seed{seed_type: seed_type}
366
+
}
367
+
end
368
+
369
+
defp seed_deploy_with_identity(seed_sid) do
370
+
%SeedDeployment{
371
+
subscription_sid: "sub_#{seed_sid}",
372
+
seed: %Seed{
373
+
sid: seed_sid,
374
+
name: "seed-#{seed_sid}",
375
+
seed_type: "nixos",
376
+
artifact: "/nix/store/#{seed_sid}"
377
+
}
269
378
}
270
379
end
271
380
end