forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
spindle/engine: timeout the entire workflow instead
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.sh>
+35
-22
+3
-3
spindle/config/config.go
+3
-3
spindle/config/config.go
···
16
16
}
17
17
18
18
type Pipelines struct {
19
-
Nixery string `env:"NIXERY, default=nixery.tangled.sh"`
20
-
StepTimeout string `env:"STEP_TIMEOUT, default=5m"`
21
-
LogDir string `env:"LOG_DIR, default=/var/log/spindle"`
19
+
Nixery string `env:"NIXERY, default=nixery.tangled.sh"`
20
+
WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"`
21
+
LogDir string `env:"LOG_DIR, default=/var/log/spindle"`
22
22
}
23
23
24
24
type Config struct {
+32
-19
spindle/engine/engine.go
+32
-19
spindle/engine/engine.go
···
102
102
defer reader.Close()
103
103
io.Copy(os.Stdout, reader)
104
104
105
+
workflowTimeoutStr := e.cfg.Pipelines.WorkflowTimeout
106
+
workflowTimeout, err := time.ParseDuration(workflowTimeoutStr)
107
+
if err != nil {
108
+
e.l.Error("failed to parse workflow timeout", "error", err, "timeout", workflowTimeoutStr)
109
+
workflowTimeout = 5 * time.Minute
110
+
}
111
+
e.l.Info("using workflow timeout", "timeout", workflowTimeout)
112
+
ctx, cancel := context.WithTimeout(ctx, workflowTimeout)
113
+
defer cancel()
114
+
105
115
err = e.StartSteps(ctx, w.Steps, wid, w.Image)
106
116
if err != nil {
107
117
if errors.Is(err, ErrTimedOut) {
···
187
177
// All other errors are bubbled up.
188
178
// Fixed version of the step execution logic
189
179
func (e *Engine) StartSteps(ctx context.Context, steps []models.Step, wid models.WorkflowId, image string) error {
190
-
stepTimeoutStr := e.cfg.Pipelines.StepTimeout
191
-
stepTimeout, err := time.ParseDuration(stepTimeoutStr)
192
-
if err != nil {
193
-
e.l.Error("failed to parse step timeout", "error", err, "timeout", stepTimeoutStr)
194
-
stepTimeout = 5 * time.Minute
195
-
}
196
-
e.l.Info("using step timeout", "timeout", stepTimeout)
197
180
198
181
for stepIdx, step := range steps {
182
+
select {
183
+
case <-ctx.Done():
184
+
return ctx.Err()
185
+
default:
186
+
}
187
+
199
188
envs := ConstructEnvs(step.Environment)
200
189
envs.AddEnv("HOME", workspaceDir)
201
190
e.l.Debug("envs for step", "step", step.Name, "envs", envs.Slice())
···
208
199
Hostname: "spindle",
209
200
Env: envs.Slice(),
210
201
}, hostConfig, nil, nil, "")
202
+
defer e.DestroyStep(ctx, resp.ID)
211
203
if err != nil {
212
204
return fmt.Errorf("creating container: %w", err)
213
205
}
···
218
208
return fmt.Errorf("connecting network: %w", err)
219
209
}
220
210
221
-
stepCtx, stepCancel := context.WithTimeout(ctx, stepTimeout)
222
-
223
-
err = e.docker.ContainerStart(stepCtx, resp.ID, container.StartOptions{})
211
+
err = e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{})
224
212
if err != nil {
225
-
stepCancel()
226
213
return err
227
214
}
228
215
e.l.Info("started container", "name", resp.ID, "step", step.Name)
···
227
220
// start tailing logs in background
228
221
tailDone := make(chan error, 1)
229
222
go func() {
230
-
tailDone <- e.TailStep(stepCtx, resp.ID, wid, stepIdx)
223
+
tailDone <- e.TailStep(ctx, resp.ID, wid, stepIdx)
231
224
}()
232
225
233
226
// wait for container completion or timeout
···
237
230
238
231
go func() {
239
232
defer close(waitDone)
240
-
state, waitErr = e.WaitStep(stepCtx, resp.ID)
233
+
state, waitErr = e.WaitStep(ctx, resp.ID)
241
234
}()
242
235
243
236
select {
···
245
238
246
239
// wait for tailing to complete
247
240
<-tailDone
248
-
stepCancel()
249
241
250
-
case <-stepCtx.Done():
251
-
e.l.Warn("step timed out; killing container", "container", resp.ID, "timeout", stepTimeout)
252
-
253
-
_ = e.DestroyStep(ctx, resp.ID)
242
+
case <-ctx.Done():
243
+
e.l.Warn("step timed out; killing container", "container", resp.ID, "step", step.Name)
244
+
err = e.DestroyStep(context.Background(), resp.ID)
245
+
if err != nil {
246
+
e.l.Error("failed to destroy step", "container", resp.ID, "error", err)
247
+
}
254
248
255
249
// wait for both goroutines to finish
256
250
<-waitDone
257
251
<-tailDone
258
252
259
-
stepCancel()
260
253
return ErrTimedOut
254
+
}
255
+
256
+
select {
257
+
case <-ctx.Done():
258
+
return ctx.Err()
259
+
default:
261
260
}
262
261
263
262
if waitErr != nil {