Deployment and lifecycle management for Nix
test: add orchestration sequence test for deployment state machine
Exercises the full server-side deployment DB state machine end-to-end:
dispatched → acknowledged → seed downloading → activating → seed
result → deployment result. Covers happy path, partial failure,
stale finalization, and seed log accumulation.
sow-75
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+253
+253
apps/sower/test/sower/orchestration/deployment_sequence_test.exs
+253
apps/sower/test/sower/orchestration/deployment_sequence_test.exs
···
1
+
defmodule Sower.Orchestration.DeploymentSequenceTest do
2
+
use Sower.DataCase, async: true
3
+
4
+
alias Sower.Orchestration.{Deployment, SeedDeployment}
5
+
6
+
import Sower.AccountsFixtures
7
+
import Sower.OrchestrationFixtures
8
+
import Sower.SeedFixtures
9
+
10
+
setup do
11
+
org = organization_fixture()
12
+
Sower.Repo.put_org_id(org.org_id)
13
+
garden = garden_fixture(%{org_id: org.org_id})
14
+
15
+
seed = seed_fixture(%{org_id: org.org_id, name: "test-seed", seed_type: "nixos"})
16
+
17
+
subscription =
18
+
subscription_fixture(%{
19
+
garden_id: garden.id,
20
+
seed_name: seed.name,
21
+
seed_type: seed.seed_type
22
+
})
23
+
24
+
deployment =
25
+
deployment_fixture(%{
26
+
org_id: org.org_id,
27
+
garden_id: garden.id,
28
+
seeds: [seed],
29
+
subscriptions: [subscription],
30
+
state: :dispatched,
31
+
last_dispatched_at: DateTime.utc_now()
32
+
})
33
+
34
+
%{org: org, garden: garden, seed: seed, subscription: subscription, deployment: deployment}
35
+
end
36
+
37
+
describe "happy path: full deployment sequence" do
38
+
test "create → dispatch → acknowledged → downloading → activating → seed success → deployment success",
39
+
%{garden: garden, seed: seed, deployment: deployment} do
40
+
# Verify initial state
41
+
assert deployment.state == :dispatched
42
+
43
+
# Step 1: Garden acknowledges deployment
44
+
{:ok, acknowledged} =
45
+
Deployment.record_deployment_status(%SowerClient.Orchestration.DeploymentStatus{
46
+
deployment_sid: deployment.sid,
47
+
status: :acknowledged
48
+
})
49
+
50
+
assert acknowledged.state == :acknowledged
51
+
52
+
# Step 2: Seed starts downloading
53
+
{:ok, _} =
54
+
SeedDeployment.record_seed_status(
55
+
%SowerClient.Orchestration.SeedDeploymentStatus{
56
+
deployment_sid: deployment.sid,
57
+
seed_sid: seed.sid,
58
+
status: :downloading
59
+
},
60
+
garden
61
+
)
62
+
63
+
assert_seed_state(deployment, seed, :downloading)
64
+
65
+
# Step 3: Seed starts activating
66
+
{:ok, _} =
67
+
SeedDeployment.record_seed_status(
68
+
%SowerClient.Orchestration.SeedDeploymentStatus{
69
+
deployment_sid: deployment.sid,
70
+
seed_sid: seed.sid,
71
+
status: :activating
72
+
},
73
+
garden
74
+
)
75
+
76
+
assert_seed_state(deployment, seed, :activating)
77
+
78
+
# Step 4: Seed result success
79
+
{:ok, _} =
80
+
SeedDeployment.record_seed_result(
81
+
%SowerClient.Orchestration.SeedDeploymentResult{
82
+
deployment_sid: deployment.sid,
83
+
seed_sid: seed.sid,
84
+
result: :success,
85
+
log: "activation complete"
86
+
},
87
+
garden
88
+
)
89
+
90
+
seed_deploy = fetch_seed_deployment(deployment, seed)
91
+
assert seed_deploy.result == :success
92
+
assert seed_deploy.log == "activation complete"
93
+
94
+
# Step 5: Deployment result success
95
+
deployed_at = DateTime.utc_now() |> DateTime.truncate(:second)
96
+
97
+
{:ok, completed} =
98
+
Deployment.record_deployment(%SowerClient.Orchestration.DeploymentResult{
99
+
request_id: SowerClient.Sid.generate("request"),
100
+
deployment_sid: deployment.sid,
101
+
result: :success,
102
+
deployed_at: deployed_at
103
+
})
104
+
105
+
assert completed.state == :completed
106
+
assert completed.result == :success
107
+
assert completed.deployed_at == deployed_at
108
+
end
109
+
end
110
+
111
+
describe "partial failure" do
112
+
setup %{org: org, garden: garden, seed: first_seed} do
113
+
second_seed =
114
+
seed_fixture(%{org_id: org.org_id, name: "test-seed-2", seed_type: "nixos"})
115
+
116
+
two_seed_deployment =
117
+
deployment_fixture(%{
118
+
org_id: org.org_id,
119
+
garden_id: garden.id,
120
+
seeds: [first_seed, second_seed],
121
+
state: :dispatched,
122
+
last_dispatched_at: DateTime.utc_now()
123
+
})
124
+
125
+
%{second_seed: second_seed, two_seed_deployment: two_seed_deployment}
126
+
end
127
+
128
+
test "one seed succeeds and one fails results in partial deployment",
129
+
%{
130
+
garden: garden,
131
+
seed: first_seed,
132
+
second_seed: second_seed,
133
+
two_seed_deployment: deployment
134
+
} do
135
+
# Acknowledge
136
+
{:ok, _} =
137
+
Deployment.record_deployment_status(%SowerClient.Orchestration.DeploymentStatus{
138
+
deployment_sid: deployment.sid,
139
+
status: :acknowledged
140
+
})
141
+
142
+
# First seed succeeds
143
+
{:ok, _} =
144
+
SeedDeployment.record_seed_result(
145
+
%SowerClient.Orchestration.SeedDeploymentResult{
146
+
deployment_sid: deployment.sid,
147
+
seed_sid: first_seed.sid,
148
+
result: :success,
149
+
log: "ok"
150
+
},
151
+
garden
152
+
)
153
+
154
+
# Second seed fails
155
+
{:ok, _} =
156
+
SeedDeployment.record_seed_result(
157
+
%SowerClient.Orchestration.SeedDeploymentResult{
158
+
deployment_sid: deployment.sid,
159
+
seed_sid: second_seed.sid,
160
+
result: :failure,
161
+
log: "activation failed"
162
+
},
163
+
garden
164
+
)
165
+
166
+
# Deployment reports partial
167
+
{:ok, completed} =
168
+
Deployment.record_deployment(%SowerClient.Orchestration.DeploymentResult{
169
+
request_id: SowerClient.Sid.generate("request"),
170
+
deployment_sid: deployment.sid,
171
+
result: :partial,
172
+
deployed_at: DateTime.utc_now() |> DateTime.truncate(:second)
173
+
})
174
+
175
+
assert completed.state == :completed
176
+
assert completed.result == :partial
177
+
178
+
# Verify individual seed results
179
+
assert fetch_seed_deployment(deployment, first_seed).result == :success
180
+
assert fetch_seed_deployment(deployment, second_seed).result == :failure
181
+
end
182
+
end
183
+
184
+
describe "stale finalization" do
185
+
test "unresolved deployment is marked stale after timeout", %{deployment: deployment} do
186
+
assert deployment.state == :dispatched
187
+
188
+
# Simulate finalization by shifting "now" 3 hours into the future
189
+
now = DateTime.add(DateTime.utc_now(), 3 * 60 * 60, :second)
190
+
{:ok, count} = Deployment.finalize_stale_deployments(now: now)
191
+
assert count == 1
192
+
193
+
stale = Deployment.get_deployment!(deployment.id)
194
+
assert stale.state == :stale
195
+
assert stale.result == :failure
196
+
assert stale.deployed_at != nil
197
+
end
198
+
end
199
+
200
+
describe "seed log accumulation" do
201
+
test "multiple seed results append logs", %{
202
+
garden: garden,
203
+
seed: seed,
204
+
deployment: deployment
205
+
} do
206
+
# First result with log but no result (progress update)
207
+
{:ok, _} =
208
+
SeedDeployment.record_seed_result(
209
+
%SowerClient.Orchestration.SeedDeploymentResult{
210
+
deployment_sid: deployment.sid,
211
+
seed_sid: seed.sid,
212
+
log: "downloading store paths"
213
+
},
214
+
garden
215
+
)
216
+
217
+
sd = fetch_seed_deployment(deployment, seed)
218
+
assert sd.log == "downloading store paths"
219
+
assert sd.result == nil
220
+
221
+
# Second result with log and final result
222
+
{:ok, _} =
223
+
SeedDeployment.record_seed_result(
224
+
%SowerClient.Orchestration.SeedDeploymentResult{
225
+
deployment_sid: deployment.sid,
226
+
seed_sid: seed.sid,
227
+
result: :success,
228
+
log: "activation complete"
229
+
},
230
+
garden
231
+
)
232
+
233
+
sd = fetch_seed_deployment(deployment, seed)
234
+
assert sd.log == "downloading store paths\nactivation complete"
235
+
assert sd.result == :success
236
+
end
237
+
end
238
+
239
+
defp assert_seed_state(deployment, seed, expected_state) do
240
+
sd = fetch_seed_deployment(deployment, seed)
241
+
assert sd.state == expected_state
242
+
end
243
+
244
+
defp fetch_seed_deployment(deployment, seed) do
245
+
Repo.one!(
246
+
from(sd in SeedDeployment,
247
+
join: s in assoc(sd, :seed),
248
+
where: sd.deployment_id == ^deployment.id and s.sid == ^seed.sid
249
+
),
250
+
skip_org_id: true
251
+
)
252
+
end
253
+
end