forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
appview,spindle: handle graceful shutdown of websockets
Signed-off-by: oppiliappan <me@oppi.li>
+100
-34
+26
-3
appview/pipelines/pipelines.go
+26
-3
appview/pipelines/pipelines.go
···
158
158
l.Error("websocket upgrade failed", "err", err)
159
159
return
160
160
}
161
-
defer clientConn.Close()
161
+
defer func() {
162
+
_ = clientConn.WriteControl(
163
+
websocket.CloseMessage,
164
+
websocket.FormatCloseMessage(websocket.CloseNormalClosure, "log stream complete"),
165
+
time.Now().Add(time.Second),
166
+
)
167
+
clientConn.Close()
168
+
}()
162
169
163
170
ctx, cancel := context.WithCancel(r.Context())
164
171
defer cancel()
···
243
236
// start a goroutine to read from spindle
244
237
go func() {
245
238
defer close(msgChan)
239
+
defer close(errChan)
240
+
246
241
for {
247
242
_, msg, err := spindleConn.ReadMessage()
248
243
if err != nil {
249
-
errChan <- err
244
+
if websocket.IsCloseError(err,
245
+
websocket.CloseNormalClosure,
246
+
websocket.CloseGoingAway,
247
+
websocket.CloseAbnormalClosure) {
248
+
errChan <- nil // signal graceful end
249
+
} else {
250
+
errChan <- err
251
+
}
250
252
return
251
253
}
252
254
msgChan <- msg
···
268
252
l.Info("client disconnected")
269
253
return
270
254
case err := <-errChan:
271
-
l.Error("error reading from spindle", "err", err)
255
+
if err != nil {
256
+
l.Error("error reading from spindle", "err", err)
257
+
}
258
+
259
+
if err == nil {
260
+
l.Info("log tail complete")
261
+
}
262
+
272
263
return
273
264
case msg := <-msgChan:
274
265
var logLine spindlemodel.LogLine
+34
spindle/db/events.go
+34
spindle/db/events.go
···
120
120
121
121
}
122
122
123
+
func (d *DB) GetStatus(workflowId models.WorkflowId) (*tangled.PipelineStatus, error) {
124
+
pipelineAtUri := workflowId.PipelineId.AtUri()
125
+
126
+
var eventJson string
127
+
err := d.QueryRow(
128
+
`
129
+
select
130
+
event from events
131
+
where
132
+
nsid = ?
133
+
and json_extract(event, '$.pipeline') = ?
134
+
and json_extract(event, '$.workflow') = ?
135
+
order by
136
+
created desc
137
+
limit
138
+
1
139
+
`,
140
+
tangled.PipelineStatusNSID,
141
+
string(pipelineAtUri),
142
+
workflowId.Name,
143
+
).Scan(&eventJson)
144
+
145
+
if err != nil {
146
+
return nil, err
147
+
}
148
+
149
+
var status tangled.PipelineStatus
150
+
if err := json.Unmarshal([]byte(eventJson), &status); err != nil {
151
+
return nil, err
152
+
}
153
+
154
+
return &status, nil
155
+
}
156
+
123
157
func (d *DB) StatusPending(workflowId models.WorkflowId, n *notifier.Notifier) error {
124
158
return d.createStatusEvent(workflowId, models.StatusKindPending, nil, nil, n)
125
159
}
+5
-1
spindle/engine/engine.go
+5
-1
spindle/engine/engine.go
···
326
326
}
327
327
defer wfLogger.Close()
328
328
329
-
_, err = stdcopy.StdCopy(wfLogger.Stdout(), wfLogger.Stderr(), logs)
329
+
_, err = stdcopy.StdCopy(
330
+
wfLogger.Writer("stdout", stepIdx),
331
+
wfLogger.Writer("stderr", stepIdx),
332
+
logs,
333
+
)
330
334
if err != nil && err != io.EOF && !errors.Is(err, context.DeadlineExceeded) {
331
335
return fmt.Errorf("failed to copy logs: %w", err)
332
336
}
+5
-23
spindle/engine/logger.go
+5
-23
spindle/engine/logger.go
···
17
17
}
18
18
19
19
func NewWorkflowLogger(baseDir string, wid models.WorkflowId) (*WorkflowLogger, error) {
20
-
dir := filepath.Join(baseDir, wid.String())
21
-
if err := os.MkdirAll(dir, 0755); err != nil {
22
-
return nil, fmt.Errorf("creating log dir: %w", err)
23
-
}
24
-
25
20
path := LogFilePath(baseDir, wid)
26
21
27
-
file, err := os.Create(path)
22
+
file, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
28
23
if err != nil {
29
24
return nil, fmt.Errorf("creating log file: %w", err)
30
25
}
···
38
43
return l.file.Close()
39
44
}
40
45
41
-
func OpenLogFile(baseDir string, workflowID models.WorkflowId) (*os.File, error) {
42
-
logPath := LogFilePath(baseDir, workflowID)
43
-
44
-
file, err := os.Open(logPath)
45
-
if err != nil {
46
-
return nil, fmt.Errorf("error opening log file: %w", err)
47
-
}
48
-
49
-
return file, nil
50
-
}
51
-
52
46
func LogFilePath(baseDir string, workflowID models.WorkflowId) string {
53
47
logFilePath := filepath.Join(baseDir, fmt.Sprintf("%s.log", workflowID.String()))
54
48
return logFilePath
55
49
}
56
50
57
-
func (l *WorkflowLogger) Stdout() io.Writer {
58
-
return &jsonWriter{logger: l, stream: "stdout"}
59
-
}
60
-
61
-
func (l *WorkflowLogger) Stderr() io.Writer {
62
-
return &jsonWriter{logger: l, stream: "stderr"}
51
+
func (l *WorkflowLogger) Writer(stream string, stepId int) io.Writer {
52
+
return &jsonWriter{logger: l, stream: stream, stepId: stepId}
63
53
}
64
54
65
55
type jsonWriter struct {
66
56
logger *WorkflowLogger
67
57
stream string
58
+
stepId int
68
59
}
69
60
70
61
func (w *jsonWriter) Write(p []byte) (int, error) {
···
59
78
entry := models.LogLine{
60
79
Stream: w.stream,
61
80
Data: line,
81
+
StepId: w.stepId,
62
82
}
63
83
64
84
if err := w.logger.encoder.Encode(entry); err != nil {
+1
spindle/models/models.go
+1
spindle/models/models.go
+29
-7
spindle/stream.go
+29
-7
spindle/stream.go
···
4
4
"context"
5
5
"encoding/json"
6
6
"fmt"
7
+
"io"
7
8
"net/http"
8
9
"strconv"
9
10
"time"
···
106
105
http.Error(w, "failed to upgrade", http.StatusInternalServerError)
107
106
return
108
107
}
109
-
defer conn.Close()
108
+
defer func() {
109
+
_ = conn.WriteControl(
110
+
websocket.CloseMessage,
111
+
websocket.FormatCloseMessage(websocket.CloseNormalClosure, "log stream complete"),
112
+
time.Now().Add(time.Second),
113
+
)
114
+
conn.Close()
115
+
}()
110
116
l.Debug("upgraded http to wss")
111
117
112
118
ctx, cancel := context.WithCancel(r.Context())
···
130
122
}()
131
123
132
124
if err := s.streamLogsFromDisk(ctx, conn, wid); err != nil {
133
-
l.Error("log stream failed", "err", err)
125
+
l.Info("log stream ended", "err", err)
134
126
}
135
-
l.Debug("logs connection closed")
127
+
128
+
l.Info("logs connection closed")
136
129
}
137
130
138
131
func (s *Spindle) streamLogsFromDisk(ctx context.Context, conn *websocket.Conn, wid models.WorkflowId) error {
132
+
status, err := s.db.GetStatus(wid)
133
+
if err != nil {
134
+
return err
135
+
}
136
+
isFinished := models.StatusKind(status.Status).IsFinish()
137
+
139
138
filePath := engine.LogFilePath(s.cfg.Pipelines.LogDir, wid)
140
139
141
140
config := tail.Config{
142
-
Follow: true,
143
-
ReOpen: true,
141
+
Follow: !isFinished,
142
+
ReOpen: !isFinished,
144
143
MustExist: false,
145
-
Location: &tail.SeekInfo{Offset: 0, Whence: 0},
146
-
Logger: tail.DiscardingLogger,
144
+
Location: &tail.SeekInfo{
145
+
Offset: 0,
146
+
Whence: io.SeekStart,
147
+
},
148
+
// Logger: tail.DiscardingLogger,
147
149
}
148
150
149
151
t, err := tail.TailFile(filePath, config)
···
167
149
case <-ctx.Done():
168
150
return ctx.Err()
169
151
case line := <-t.Lines:
152
+
if line == nil && isFinished {
153
+
return fmt.Errorf("tail completed")
154
+
}
155
+
170
156
if line == nil {
171
157
return fmt.Errorf("tail channel closed unexpectedly")
172
158
}