forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
spindle/queue: enqueue pipeline jobs
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.sh>
+70
-13
+37
spindle/queue/queue.go
+37
spindle/queue/queue.go
···
1
+
package queue
2
+
3
+
type Job struct {
4
+
Run func() error
5
+
OnFail func(error)
6
+
}
7
+
8
+
type Queue struct {
9
+
jobs chan Job
10
+
}
11
+
12
+
func NewQueue(size int) *Queue {
13
+
return &Queue{
14
+
jobs: make(chan Job, size),
15
+
}
16
+
}
17
+
18
+
func (q *Queue) Enqueue(job Job) bool {
19
+
select {
20
+
case q.jobs <- job:
21
+
return true
22
+
default:
23
+
return false
24
+
}
25
+
}
26
+
27
+
func (q *Queue) StartRunner() {
28
+
go func() {
29
+
for job := range q.jobs {
30
+
if err := job.Run(); err != nil {
31
+
if job.OnFail != nil {
32
+
job.OnFail(err)
33
+
}
34
+
}
35
+
}
36
+
}()
37
+
}
+33
-13
spindle/server.go
+33
-13
spindle/server.go
···
1
1
package spindle
2
2
3
3
import (
4
+
"context"
4
5
"encoding/json"
5
6
"fmt"
6
7
"log/slog"
7
8
"net/http"
8
9
9
10
"github.com/go-chi/chi/v5"
10
-
"golang.org/x/net/context"
11
11
"tangled.sh/tangled.sh/core/api/tangled"
12
12
"tangled.sh/tangled.sh/core/jetstream"
13
13
"tangled.sh/tangled.sh/core/knotclient"
···
17
17
"tangled.sh/tangled.sh/core/spindle/config"
18
18
"tangled.sh/tangled.sh/core/spindle/db"
19
19
"tangled.sh/tangled.sh/core/spindle/engine"
20
+
"tangled.sh/tangled.sh/core/spindle/queue"
20
21
)
21
22
22
23
type Spindle struct {
···
27
26
l *slog.Logger
28
27
n *notifier.Notifier
29
28
eng *engine.Engine
29
+
jq *queue.Queue
30
30
}
31
31
32
32
func Run(ctx context.Context) error {
···
60
58
return err
61
59
}
62
60
61
+
jq := queue.NewQueue(100)
62
+
63
+
// starts a job queue runner in the background
64
+
jq.StartRunner()
65
+
63
66
spindle := Spindle{
64
67
jc: jc,
65
68
e: e,
···
72
65
l: logger,
73
66
n: &n,
74
67
eng: eng,
68
+
jq: jq,
75
69
}
76
70
71
+
// for each incoming sh.tangled.pipeline, we execute
72
+
// spindle.processPipeline, which in turn enqueues the pipeline
73
+
// job in the above registered queue.
77
74
go func() {
78
75
logger.Info("starting event consumer")
79
76
knotEventSource := knotclient.NewEventSource("localhost:5555")
···
85
74
ccfg := knotclient.NewConsumerConfig()
86
75
ccfg.Logger = logger
87
76
ccfg.Dev = cfg.Server.Dev
88
-
ccfg.ProcessFunc = spindle.exec
77
+
ccfg.ProcessFunc = spindle.processPipeline
89
78
ccfg.AddEventSource(knotEventSource)
90
79
91
80
ec := knotclient.NewEventConsumer(*ccfg)
···
107
96
return mux
108
97
}
109
98
110
-
func (s *Spindle) exec(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error {
99
+
func (s *Spindle) processPipeline(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error {
111
100
if msg.Nsid == tangled.PipelineNSID {
112
101
pipeline := tangled.Pipeline{}
113
102
err := json.Unmarshal(msg.EventJson, &pipeline)
···
116
105
return err
117
106
}
118
107
119
-
// this is a "fake" at uri for now
120
-
pipelineAtUri := fmt.Sprintf("at://%s/did:web:%s/%s", tangled.PipelineNSID, pipeline.TriggerMetadata.Repo.Knot, msg.Rkey)
108
+
ok := s.jq.Enqueue(queue.Job{
109
+
Run: func() error {
110
+
// this is a "fake" at uri for now
111
+
pipelineAtUri := fmt.Sprintf("at://%s/did:web:%s/%s", tangled.PipelineNSID, pipeline.TriggerMetadata.Repo.Knot, msg.Rkey)
121
112
122
-
rkey := TID()
123
-
err = s.eng.SetupPipeline(ctx, &pipeline, pipelineAtUri, rkey)
124
-
if err != nil {
125
-
return err
126
-
}
127
-
err = s.eng.StartWorkflows(ctx, &pipeline, rkey)
128
-
if err != nil {
129
-
return err
113
+
rkey := TID()
114
+
err = s.eng.SetupPipeline(ctx, &pipeline, pipelineAtUri, rkey)
115
+
if err != nil {
116
+
return err
117
+
}
118
+
return s.eng.StartWorkflows(ctx, &pipeline, rkey)
119
+
},
120
+
OnFail: func(error) {
121
+
s.l.Error("pipeline setup failed", "error", err)
122
+
},
123
+
})
124
+
if ok {
125
+
s.l.Info("pipeline enqueued successfully", "id", msg.Rkey)
126
+
} else {
127
+
s.l.Error("failed to enqueue pipeline: queue is full")
130
128
}
131
129
}
132
130