forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
spindle/engine: stream logs to websocket
Logs are streamed for each running pipeline on a websocket at
/logs/{pipelineID}. engine.TailStep demuxes stdout and stderr from the
container's logs and pipes that out to corresponding stdout and stderr
channels.
These channels are maintained inside engine's container
struct, key'd by the pipeline ID, and protected by a read/write mutex.
engine.LogChannels fetches the stdout/stderr chans as recieve-only if
the pipeline is known to exist.
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.sh>
+201
-7
+98
-5
spindle/engine/engine.go
+98
-5
spindle/engine/engine.go
···
1
1
package engine
2
2
3
3
import (
4
+
"bufio"
4
5
"context"
5
6
"fmt"
6
7
"io"
···
33
32
l *slog.Logger
34
33
db *db.DB
35
34
n *notifier.Notifier
35
+
36
+
chanMu sync.RWMutex
37
+
stdoutChans map[string]chan string
38
+
stderrChans map[string]chan string
36
39
}
37
40
38
41
func New(ctx context.Context, db *db.DB, n *notifier.Notifier) (*Engine, error) {
···
47
42
48
43
l := log.FromContext(ctx).With("component", "spindle")
49
44
50
-
return &Engine{docker: dcli, l: l, db: db, n: n}, nil
45
+
e := &Engine{
46
+
docker: dcli,
47
+
l: l,
48
+
db: db,
49
+
n: n,
50
+
}
51
+
52
+
e.stdoutChans = make(map[string]chan string, 100)
53
+
e.stderrChans = make(map[string]chan string, 100)
54
+
55
+
return e, nil
51
56
}
52
57
53
58
// SetupPipeline sets up a new network for the pipeline, and possibly volumes etc.
···
151
136
// ONLY marks pipeline as failed if container's exit code is non-zero.
152
137
// All other errors are bubbled up.
153
138
func (e *Engine) StartSteps(ctx context.Context, steps []*tangled.Pipeline_Step, id, image string) error {
139
+
// set up logging channels
140
+
e.chanMu.Lock()
141
+
if _, exists := e.stdoutChans[id]; !exists {
142
+
e.stdoutChans[id] = make(chan string, 100)
143
+
}
144
+
if _, exists := e.stderrChans[id]; !exists {
145
+
e.stderrChans[id] = make(chan string, 100)
146
+
}
147
+
e.chanMu.Unlock()
148
+
149
+
// close channels after all steps are complete
150
+
defer func() {
151
+
close(e.stdoutChans[id])
152
+
close(e.stderrChans[id])
153
+
}()
154
+
154
155
for _, step := range steps {
155
156
hostConfig := hostConfig(id)
156
157
resp, err := e.docker.ContainerCreate(ctx, &container.Config{
···
197
166
wg.Add(1)
198
167
go func() {
199
168
defer wg.Done()
200
-
err := e.TailStep(ctx, resp.ID)
169
+
err := e.TailStep(ctx, resp.ID, id)
201
170
if err != nil {
202
171
e.l.Error("failed to tail container", "container", resp.ID)
203
172
return
···
242
211
return info.State, nil
243
212
}
244
213
245
-
func (e *Engine) TailStep(ctx context.Context, containerID string) error {
214
+
func (e *Engine) TailStep(ctx context.Context, containerID, pipelineID string) error {
246
215
logs, err := e.docker.ContainerLogs(ctx, containerID, container.LogsOptions{
247
216
Follow: true,
248
217
ShowStdout: true,
···
254
223
return err
255
224
}
256
225
226
+
// using StdCopy we demux logs and stream stdout and stderr to different
227
+
// channels.
228
+
//
229
+
// stdout w||r stdoutCh
230
+
// stderr w||r stderrCh
231
+
//
232
+
233
+
rpipeOut, wpipeOut := io.Pipe()
234
+
rpipeErr, wpipeErr := io.Pipe()
235
+
257
236
go func() {
258
-
_, _ = stdcopy.StdCopy(os.Stdout, os.Stdout, logs)
259
-
_ = logs.Close()
237
+
defer wpipeOut.Close()
238
+
defer wpipeErr.Close()
239
+
_, err := stdcopy.StdCopy(wpipeOut, wpipeErr, logs)
240
+
if err != nil && err != io.EOF {
241
+
e.l.Error("failed to copy logs", "error", err)
242
+
}
260
243
}()
244
+
245
+
// read from stdout and send to stdout pipe
246
+
// NOTE: the stdoutCh channnel is closed further up in StartSteps
247
+
// once all steps are done.
248
+
go func() {
249
+
e.chanMu.RLock()
250
+
stdoutCh := e.stdoutChans[pipelineID]
251
+
e.chanMu.RUnlock()
252
+
253
+
scanner := bufio.NewScanner(rpipeOut)
254
+
for scanner.Scan() {
255
+
stdoutCh <- scanner.Text()
256
+
}
257
+
if err := scanner.Err(); err != nil {
258
+
e.l.Error("failed to scan stdout", "error", err)
259
+
}
260
+
}()
261
+
262
+
// read from stderr and send to stderr pipe
263
+
// NOTE: the stderrCh channnel is closed further up in StartSteps
264
+
// once all steps are done.
265
+
go func() {
266
+
e.chanMu.RLock()
267
+
stderrCh := e.stderrChans[pipelineID]
268
+
e.chanMu.RUnlock()
269
+
270
+
scanner := bufio.NewScanner(rpipeErr)
271
+
for scanner.Scan() {
272
+
stderrCh <- scanner.Text()
273
+
}
274
+
if err := scanner.Err(); err != nil {
275
+
e.l.Error("failed to scan stderr", "error", err)
276
+
}
277
+
}()
278
+
261
279
return nil
280
+
}
281
+
282
+
func (e *Engine) LogChannels(pipelineID string) (stdout <-chan string, stderr <-chan string, ok bool) {
283
+
e.chanMu.RLock()
284
+
defer e.chanMu.RUnlock()
285
+
286
+
stdoutCh, ok1 := e.stdoutChans[pipelineID]
287
+
stderrCh, ok2 := e.stderrChans[pipelineID]
288
+
289
+
if !ok1 || !ok2 {
290
+
return nil, nil, false
291
+
}
292
+
return stdoutCh, stderrCh, true
262
293
}
263
294
264
295
func workspaceVolume(id string) string {
+3
-2
spindle/server.go
+3
-2
spindle/server.go
···
6
6
"log/slog"
7
7
"net/http"
8
8
9
+
"github.com/go-chi/chi/v5"
9
10
"golang.org/x/net/context"
10
11
"tangled.sh/tangled.sh/core/api/tangled"
11
12
"tangled.sh/tangled.sh/core/jetstream"
···
53
52
}
54
53
55
54
n := notifier.New()
56
-
57
55
eng, err := engine.New(ctx, d, &n)
58
56
if err != nil {
59
57
return err
···
89
89
}
90
90
91
91
func (s *Spindle) Router() http.Handler {
92
-
mux := &http.ServeMux{}
92
+
mux := chi.NewRouter()
93
93
94
94
mux.HandleFunc("/events", s.Events)
95
+
mux.HandleFunc("/logs/{pipelineID}", s.Logs)
95
96
return mux
96
97
}
97
98
+100
spindle/stream.go
+100
spindle/stream.go
···
1
1
package spindle
2
2
3
3
import (
4
+
"fmt"
4
5
"net/http"
5
6
"time"
6
7
7
8
"context"
8
9
10
+
"github.com/go-chi/chi/v5"
9
11
"github.com/gorilla/websocket"
10
12
)
11
13
···
74
72
}
75
73
}
76
74
}
75
+
}
76
+
77
+
func (s *Spindle) Logs(w http.ResponseWriter, r *http.Request) {
78
+
l := s.l.With("handler", "Logs")
79
+
80
+
pipelineID := chi.URLParam(r, "pipelineID")
81
+
if pipelineID == "" {
82
+
http.Error(w, "pipelineID required", http.StatusBadRequest)
83
+
return
84
+
}
85
+
l = l.With("pipelineID", pipelineID)
86
+
87
+
conn, err := upgrader.Upgrade(w, r, nil)
88
+
if err != nil {
89
+
l.Error("websocket upgrade failed", "err", err)
90
+
http.Error(w, "failed to upgrade", http.StatusInternalServerError)
91
+
return
92
+
}
93
+
defer conn.Close()
94
+
l.Info("upgraded http to wss")
95
+
96
+
ctx, cancel := context.WithCancel(r.Context())
97
+
defer cancel()
98
+
99
+
go func() {
100
+
for {
101
+
if _, _, err := conn.NextReader(); err != nil {
102
+
l.Info("client disconnected", "err", err)
103
+
cancel()
104
+
return
105
+
}
106
+
}
107
+
}()
108
+
109
+
if err := s.streamLogs(ctx, conn, pipelineID); err != nil {
110
+
l.Error("streamLogs failed", "err", err)
111
+
}
112
+
l.Info("logs connection closed")
113
+
}
114
+
115
+
func (s *Spindle) streamLogs(ctx context.Context, conn *websocket.Conn, pipelineID string) error {
116
+
l := s.l.With("pipelineID", pipelineID)
117
+
118
+
stdoutCh, stderrCh, ok := s.eng.LogChannels(pipelineID)
119
+
if !ok {
120
+
return fmt.Errorf("pipelineID %q not found", pipelineID)
121
+
}
122
+
123
+
done := make(chan struct{})
124
+
125
+
go func() {
126
+
for {
127
+
select {
128
+
case line, ok := <-stdoutCh:
129
+
if !ok {
130
+
done <- struct{}{}
131
+
return
132
+
}
133
+
msg := map[string]string{"type": "stdout", "data": line}
134
+
if err := conn.WriteJSON(msg); err != nil {
135
+
l.Error("write stdout failed", "err", err)
136
+
done <- struct{}{}
137
+
return
138
+
}
139
+
case <-ctx.Done():
140
+
done <- struct{}{}
141
+
return
142
+
}
143
+
}
144
+
}()
145
+
146
+
go func() {
147
+
for {
148
+
select {
149
+
case line, ok := <-stderrCh:
150
+
if !ok {
151
+
done <- struct{}{}
152
+
return
153
+
}
154
+
msg := map[string]string{"type": "stderr", "data": line}
155
+
if err := conn.WriteJSON(msg); err != nil {
156
+
l.Error("write stderr failed", "err", err)
157
+
done <- struct{}{}
158
+
return
159
+
}
160
+
case <-ctx.Done():
161
+
done <- struct{}{}
162
+
return
163
+
}
164
+
}
165
+
}()
166
+
167
+
select {
168
+
case <-done:
169
+
case <-ctx.Done():
170
+
}
171
+
172
+
return nil
77
173
}
78
174
79
175
func (s *Spindle) streamPipelines(conn *websocket.Conn, cursor *string) error {