forked from
tangled.org/core
this repo has no description
appview: stream logs from workflow endpoint
Signed-off-by: oppiliappan <me@oppi.li>
+254
-44
+23
-9
appview/db/pipeline.go
+23
-9
appview/db/pipeline.go
···
27
27
}
28
28
29
29
type WorkflowStatus struct {
30
-
data []PipelineStatus
30
+
Data []PipelineStatus
31
31
}
32
32
33
33
func (w WorkflowStatus) Latest() PipelineStatus {
34
-
return w.data[len(w.data)-1]
34
+
return w.Data[len(w.Data)-1]
35
35
}
36
36
37
37
// time taken by this workflow to reach an "end state"
38
38
func (w WorkflowStatus) TimeTaken() time.Duration {
39
39
var start, end *time.Time
40
-
for _, s := range w.data {
40
+
for _, s := range w.Data {
41
41
if s.Status.IsStart() {
42
42
start = &s.Created
43
43
}
···
76
76
}
77
77
slices.Sort(ws)
78
78
return ws
79
+
}
80
+
81
+
// if we know that a spindle has picked up this pipeline, then it is Responding
82
+
func (p Pipeline) IsResponding() bool {
83
+
return len(p.Statuses) != 0
79
84
}
80
85
81
86
type Trigger struct {
···
256
261
status.Status,
257
262
status.Error,
258
263
status.ExitCode,
264
+
status.Created.Format(time.RFC3339),
259
265
}
260
266
261
267
placeholders := make([]string, len(args))
···
272
278
workflow,
273
279
status,
274
280
error,
275
-
exit_code
281
+
exit_code,
282
+
created
276
283
) values (%s)
277
284
`, strings.Join(placeholders, ","))
278
285
···
355
362
return nil, err
356
363
}
357
364
358
-
// Parse created time manually
359
365
p.Created, err = time.Parse(time.RFC3339, created)
360
366
if err != nil {
361
367
return nil, fmt.Errorf("invalid pipeline created timestamp %q: %w", created, err)
362
368
}
363
369
364
-
// Link trigger to pipeline
365
370
t.Id = p.TriggerId
366
371
p.Trigger = &t
367
372
p.Statuses = make(map[string]WorkflowStatus)
···
440
445
}
441
446
442
447
// append
443
-
statuses.data = append(statuses.data, ps)
448
+
statuses.Data = append(statuses.Data, ps)
444
449
445
450
// reassign
446
451
pipeline.Statuses[ps.Workflow] = statuses
···
450
455
var all []Pipeline
451
456
for _, p := range pipelines {
452
457
for _, s := range p.Statuses {
453
-
slices.SortFunc(s.data, func(a, b PipelineStatus) int {
458
+
slices.SortFunc(s.Data, func(a, b PipelineStatus) int {
454
459
if a.Created.After(b.Created) {
455
460
return 1
456
461
}
457
-
return -1
462
+
if a.Created.Before(b.Created) {
463
+
return -1
464
+
}
465
+
if a.ID > b.ID {
466
+
return 1
467
+
}
468
+
if a.ID < b.ID {
469
+
return -1
470
+
}
471
+
return 0
458
472
})
459
473
}
460
474
all = append(all, p)
+1
appview/pages/pages.go
+1
appview/pages/pages.go
+1
appview/pages/repoinfo/repoinfo.go
+1
appview/pages/repoinfo/repoinfo.go
+1
appview/pages/templates/layouts/base.html
+1
appview/pages/templates/layouts/base.html
···
9
9
/>
10
10
<meta name="htmx-config" content='{"includeIndicatorStyles": false}'>
11
11
<script src="/static/htmx.min.js"></script>
12
+
<script src="https://cdn.jsdelivr.net/npm/htmx-ext-ws@2.0.2"></script>
12
13
<link rel="stylesheet" href="/static/tw.css?{{ cssContentHash }}" type="text/css" />
13
14
<title>{{ block "title" . }}{{ end }} · tangled</title>
14
15
{{ block "extrameta" . }}{{ end }}
+15
-2
appview/pages/templates/repo/pipelines/fragments/pipelineSymbol.html
+15
-2
appview/pages/templates/repo/pipelines/fragments/pipelineSymbol.html
···
4
4
{{ $statuses := .Statuses }}
5
5
{{ $total := len $statuses }}
6
6
{{ $success := index $c "success" }}
7
+
{{ $fail := index $c "failed" }}
8
+
{{ $empty := eq $total 0 }}
7
9
{{ $allPass := eq $success $total }}
10
+
{{ $allFail := eq $fail $total }}
8
11
9
-
{{ if $allPass }}
12
+
{{ if $empty }}
13
+
<div class="flex gap-1 items-center">
14
+
{{ i "hourglass" "size-4 text-gray-600 dark:text-gray-400 " }}
15
+
<span>0/{{ $total }}</span>
16
+
</div>
17
+
{{ else if $allPass }}
10
18
<div class="flex gap-1 items-center">
11
-
{{ i "check" "size-4 text-green-600 dark:text-green-400 " }}
19
+
{{ i "check" "size-4 text-green-600" }}
12
20
<span>{{ $total }}/{{ $total }}</span>
21
+
</div>
22
+
{{ else if $allFail }}
23
+
<div class="flex gap-1 items-center">
24
+
{{ i "x" "size-4 text-red-600" }}
25
+
<span>0/{{ $total }}</span>
13
26
</div>
14
27
{{ else }}
15
28
{{ $radius := f64 8 }}
+5
appview/pages/templates/repo/pipelines/fragments/tooltip.html
+5
appview/pages/templates/repo/pipelines/fragments/tooltip.html
+23
-9
appview/pages/templates/repo/pipelines/pipelines.html
+23
-9
appview/pages/templates/repo/pipelines/pipelines.html
···
41
41
{{ $root := index . 0 }}
42
42
{{ $p := index . 1 }}
43
43
{{ with $p }}
44
-
<div class="grid grid-cols-4 md:grid-cols-8 gap-2 items-center w-full">
45
-
<div class="col-span-1 md:col-span-5 flex items-center gap-4">
44
+
<div class="grid grid-cols-6 md:grid-cols-12 gap-2 items-center w-full">
45
+
<div class="col-span-2 md:col-span-8 flex items-center gap-4">
46
46
{{ $target := .Trigger.TargetRef }}
47
47
{{ $workflows := .Workflows }}
48
+
{{ $link := "" }}
49
+
{{ if .IsResponding }}
50
+
{{ $link = printf "/%s/pipelines/%s/workflow/%d" $root.RepoInfo.FullName .Id (index $workflows 0) }}
51
+
{{ end }}
48
52
{{ if .Trigger.IsPush }}
49
-
<a href="/{{ $root.RepoInfo.FullName }}/pipelines/{{ .Id }}/workflow/{{ index $workflows 0 }}" class="block">
50
-
<span class="font-bold">{{ $target }}</span>
51
-
<span>push</span>
52
-
</a>
53
+
<span class="font-bold">{{ $target }}</span>
54
+
<span>push</span>
53
55
<span class="hidden md:inline-flex gap-2 items-center font-mono text-sm">
54
56
{{ $old := deref .Trigger.PushOldSha }}
55
57
{{ $new := deref .Trigger.PushNewSha }}
···
70
72
{{ end }}
71
73
</div>
72
74
73
-
<div class="col-span-1 pl-4">
75
+
<div class="text-sm md:text-base col-span-1">
74
76
{{ template "repo/pipelines/fragments/pipelineSymbolLong" . }}
75
77
</div>
76
78
77
-
<div class="col-span-1 text-right">
79
+
<div class="text-sm md:text-base col-span-1 text-right">
78
80
<time title="{{ .Created | longTimeFmt }}">
79
81
{{ .Created | shortTimeFmt }} ago
80
82
</time>
81
83
</div>
82
84
83
85
{{ $t := .TimeTaken }}
84
-
<div class="col-span-1 text-right">
86
+
<div class="text-sm md:text-base col-span-1 text-right">
85
87
{{ if $t }}
86
88
<time title="{{ $t }}">{{ $t | durationFmt }}</time>
87
89
{{ else }}
88
90
<time>--</time>
89
91
{{ end }}
90
92
</div>
93
+
94
+
<div class="col-span-1 flex justify-end">
95
+
{{ if $link }}
96
+
<a class="md:hidden" href="/{{ $root.RepoInfo.FullName }}/pipelines/{{ .Id }}/workflow/{{ index $workflows 0 }}">
97
+
{{ i "arrow-up-right" "size-4" }}
98
+
</a>
99
+
<a class="hidden md:inline underline" href="/{{ $root.RepoInfo.FullName }}/pipelines/{{ .Id }}/workflow/{{ index $workflows 0 }}">
100
+
view
101
+
</a>
102
+
{{ end }}
103
+
</div>
104
+
91
105
</div>
92
106
{{ end }}
93
107
{{ end }}
+35
-18
appview/pages/templates/repo/pipelines/workflow.html
+35
-18
appview/pages/templates/repo/pipelines/workflow.html
···
23
23
{{ define "sidebar" }}
24
24
{{ $active := .Workflow }}
25
25
{{ with .Pipeline }}
26
-
<div class="rounded border border-gray-200 dark:border-gray-700">
26
+
{{ $id := .Id }}
27
+
<div class="grid grid-cols-1 rounded border border-gray-200 dark:border-gray-700 divide-y divide-gray-200 dark:divide-gray-700">
27
28
{{ range $name, $all := .Statuses }}
28
-
<div class="flex items-center justify-between p-2 border-b border-gray-200 dark:border-gray-700 {{if eq $name $active}}bg-gray-100/50 dark:bg-gray-700/50{{end}}">
29
-
{{ $lastStatus := $all.Latest }}
30
-
{{ $kind := $lastStatus.Status.String }}
29
+
<a href="/{{ $.RepoInfo.FullName }}/pipelines/{{ $id }}/workflow/{{ $name }}" class="no-underline hover:no-underline hover:bg-gray-100/25 hover:dark:bg-gray-700/25">
30
+
<div
31
+
class="flex gap-2 items-center justify-between p-2 {{ if eq $name $active }}bg-gray-100/50 dark:bg-gray-700/50{{ end }}">
32
+
{{ $lastStatus := $all.Latest }}
33
+
{{ $kind := $lastStatus.Status.String }}
34
+
35
+
{{ $t := .TimeTaken }}
36
+
{{ $time := "" }}
31
37
32
-
{{ $t := .TimeTaken }}
33
-
{{ $time := "" }}
34
-
{{ if $t }}
38
+
{{ if $t }}
35
39
{{ $time = durationFmt $t }}
36
-
{{ else }}
37
-
{{ $time = printf "%s ago" (shortTimeFmt $.Created) }}
38
-
{{ end }}
40
+
{{ else }}
41
+
{{ $time = printf "%s ago" (shortTimeFmt $lastStatus.Created) }}
42
+
{{ end }}
39
43
40
-
<div id="left" class="flex items-center gap-2 flex-shrink-0">
41
-
{{ template "repo/pipelines/fragments/workflowSymbol" $all }}
42
-
{{ $name }}
44
+
<div id="left" class="flex items-center gap-2 flex-shrink-0">
45
+
{{ template "repo/pipelines/fragments/workflowSymbol" $all }}
46
+
{{ $name }}
47
+
</div>
48
+
<div id="right" class="flex items-center gap-2 flex-shrink-0">
49
+
<span class="font-bold">{{ $kind }}</span>
50
+
<time>{{ $time }}</time>
51
+
</div>
43
52
</div>
44
-
<div id="right" class="flex items-center gap-2 flex-shrink-0">
45
-
<span class="font-bold">{{ $kind }}</span>
46
-
<time>{{ $time }}</time>
47
-
</div>
48
-
</div>
53
+
</a>
49
54
{{ end }}
50
55
</div>
51
56
{{ end }}
52
57
{{ end }}
58
+
59
+
{{ define "logs" }}
60
+
<div id="log-stream"
61
+
class="p-2 bg-gray-100 dark:bg-gray-900 font-mono text-sm min-h-96 max-h-screen overflow-auto flex flex-col-reverse [overflow-anchor:auto_!important]"
62
+
hx-ext="ws"
63
+
ws-connect="/{{ $.RepoInfo.FullName }}/pipelines/{{ .Pipeline.Id }}/workflow/{{ .Workflow }}/logs">
64
+
<div id="lines">
65
+
<!-- Each log line should be rendered with class="item" like below -->
66
+
<!-- <div class="item">[INFO] Log line here</div> -->
67
+
</div>
68
+
</div>
69
+
{{ end }}
+147
-5
appview/pipelines/pipelines.go
+147
-5
appview/pipelines/pipelines.go
···
1
1
package pipelines
2
2
3
3
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
4
7
"log/slog"
5
8
"net/http"
9
+
"strings"
10
+
"time"
6
11
7
12
"tangled.sh/tangled.sh/core/appview/config"
8
13
"tangled.sh/tangled.sh/core/appview/db"
···
13
18
"tangled.sh/tangled.sh/core/eventconsumer"
14
19
"tangled.sh/tangled.sh/core/log"
15
20
"tangled.sh/tangled.sh/core/rbac"
21
+
spindlemodel "tangled.sh/tangled.sh/core/spindle/models"
16
22
17
23
"github.com/go-chi/chi/v5"
24
+
"github.com/gorilla/websocket"
18
25
"github.com/posthog/posthog-go"
19
26
)
20
27
···
28
35
db *db.DB
29
36
enforcer *rbac.Enforcer
30
37
posthog posthog.Client
31
-
Logger *slog.Logger
38
+
logger *slog.Logger
32
39
}
33
40
34
41
func New(
···
53
60
db: db,
54
61
posthog: posthog,
55
62
enforcer: enforcer,
56
-
Logger: logger,
63
+
logger: logger,
57
64
}
58
65
}
59
66
60
67
func (p *Pipelines) Index(w http.ResponseWriter, r *http.Request) {
61
68
user := p.oauth.GetUser(r)
62
-
l := p.Logger.With("handler", "Index")
69
+
l := p.logger.With("handler", "Index")
63
70
64
71
f, err := p.repoResolver.Resolve(r)
65
72
if err != nil {
···
89
96
90
97
func (p *Pipelines) Workflow(w http.ResponseWriter, r *http.Request) {
91
98
user := p.oauth.GetUser(r)
92
-
l := p.Logger.With("handler", "Workflow")
99
+
l := p.logger.With("handler", "Workflow")
93
100
94
101
f, err := p.repoResolver.Resolve(r)
95
102
if err != nil {
···
106
113
}
107
114
108
115
workflow := chi.URLParam(r, "workflow")
109
-
if pipelineId == "" {
116
+
if workflow == "" {
110
117
l.Error("empty workflow name")
111
118
return
112
119
}
···
137
144
Workflow: workflow,
138
145
})
139
146
}
147
+
148
+
var upgrader = websocket.Upgrader{
149
+
ReadBufferSize: 1024,
150
+
WriteBufferSize: 1024,
151
+
}
152
+
153
+
func (p *Pipelines) Logs(w http.ResponseWriter, r *http.Request) {
154
+
l := p.logger.With("handler", "logs")
155
+
156
+
clientConn, err := upgrader.Upgrade(w, r, nil)
157
+
if err != nil {
158
+
l.Error("websocket upgrade failed", "err", err)
159
+
return
160
+
}
161
+
defer clientConn.Close()
162
+
163
+
ctx, cancel := context.WithCancel(r.Context())
164
+
defer cancel()
165
+
go func() {
166
+
for {
167
+
if _, _, err := clientConn.NextReader(); err != nil {
168
+
l.Error("failed to read", "err", err)
169
+
cancel()
170
+
return
171
+
}
172
+
}
173
+
}()
174
+
175
+
user := p.oauth.GetUser(r)
176
+
f, err := p.repoResolver.Resolve(r)
177
+
if err != nil {
178
+
l.Error("failed to get repo and knot", "err", err)
179
+
http.Error(w, "bad repo/knot", http.StatusBadRequest)
180
+
return
181
+
}
182
+
183
+
repoInfo := f.RepoInfo(user)
184
+
185
+
pipelineId := chi.URLParam(r, "pipeline")
186
+
workflow := chi.URLParam(r, "workflow")
187
+
if pipelineId == "" || workflow == "" {
188
+
http.Error(w, "missing pipeline ID or workflow", http.StatusBadRequest)
189
+
return
190
+
}
191
+
192
+
ps, err := db.GetPipelineStatuses(
193
+
p.db,
194
+
db.FilterEq("repo_owner", repoInfo.OwnerDid),
195
+
db.FilterEq("repo_name", repoInfo.Name),
196
+
db.FilterEq("knot", repoInfo.Knot),
197
+
db.FilterEq("id", pipelineId),
198
+
)
199
+
if err != nil || len(ps) != 1 {
200
+
l.Error("pipeline query failed", "err", err, "count", len(ps))
201
+
http.Error(w, "pipeline not found", http.StatusNotFound)
202
+
return
203
+
}
204
+
205
+
singlePipeline := ps[0]
206
+
spindle := repoInfo.Spindle
207
+
knot := repoInfo.Knot
208
+
rkey := singlePipeline.Rkey
209
+
210
+
if spindle == "" || knot == "" || rkey == "" {
211
+
http.Error(w, "invalid repo info", http.StatusBadRequest)
212
+
return
213
+
}
214
+
215
+
scheme := "wss"
216
+
if p.config.Core.Dev {
217
+
scheme = "ws"
218
+
}
219
+
220
+
url := scheme + "://" + strings.Join([]string{spindle, "logs", knot, rkey, workflow}, "/")
221
+
l = l.With("url", url)
222
+
l.Info("logs endpoint hit")
223
+
224
+
spindleConn, _, err := websocket.DefaultDialer.Dial(url, nil)
225
+
if err != nil {
226
+
l.Error("websocket dial failed", "err", err)
227
+
http.Error(w, "failed to connect to log stream", http.StatusBadGateway)
228
+
return
229
+
}
230
+
defer spindleConn.Close()
231
+
232
+
// create a channel for incoming messages
233
+
msgChan := make(chan []byte, 10)
234
+
errChan := make(chan error, 1)
235
+
236
+
// start a goroutine to read from spindle
237
+
go func() {
238
+
defer close(msgChan)
239
+
for {
240
+
_, msg, err := spindleConn.ReadMessage()
241
+
if err != nil {
242
+
errChan <- err
243
+
return
244
+
}
245
+
msgChan <- msg
246
+
}
247
+
}()
248
+
249
+
for {
250
+
select {
251
+
case <-ctx.Done():
252
+
l.Info("client disconnected")
253
+
return
254
+
case err := <-errChan:
255
+
l.Error("error reading from spindle", "err", err)
256
+
return
257
+
case msg := <-msgChan:
258
+
var logLine spindlemodel.LogLine
259
+
if err = json.Unmarshal(msg, &logLine); err != nil {
260
+
l.Error("failed to parse logline", "err", err)
261
+
continue
262
+
}
263
+
264
+
html := fmt.Appendf(nil, `
265
+
<div id="lines" hx-swap-oob="beforeend">
266
+
<p>%s: %s</p>
267
+
</div>
268
+
`, logLine.Stream, logLine.Data)
269
+
270
+
if err = clientConn.WriteMessage(websocket.TextMessage, html); err != nil {
271
+
l.Error("error writing to client", "err", err)
272
+
return
273
+
}
274
+
case <-time.After(30 * time.Second):
275
+
l.Debug("sent keepalive")
276
+
if err = clientConn.WriteControl(websocket.PingMessage, []byte{}, time.Now().Add(time.Second)); err != nil {
277
+
l.Error("failed to write control", "err", err)
278
+
}
279
+
}
280
+
}
281
+
}
+1
appview/pipelines/router.go
+1
appview/pipelines/router.go
+1
appview/reporesolver/resolver.go
+1
appview/reporesolver/resolver.go
+1
-1
flake.nix
+1
-1
flake.nix
···
59
59
inherit (gitignore.lib) gitignoreSource;
60
60
in {
61
61
overlays.default = final: prev: let
62
-
goModHash = "sha256-G+59ZwQwBbnO9ZjAB5zMEmWZbeG4k7ko/lPz+ceqYKs=";
62
+
goModHash = "sha256-2RUwj16RNaZ/gCOcd7b3LRCHiROCRj9HuzbBdLdgWGo=";
63
63
appviewDeps = {
64
64
inherit htmx-src htmx-ws-src lucide-src inter-fonts-src ibm-plex-mono-src goModHash gitignoreSource;
65
65
};