forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
eventconsumer: extract knotclient/events into its own package
Signed-off-by: oppiliappan <me@oppi.li>
authored by
oppiliappan
and committed by
Tangled
85720a2c
fc4ba7b2
+137
-63
+39
eventconsumer/knot.go
+39
eventconsumer/knot.go
···
1
+
package eventconsumer
2
+
3
+
import (
4
+
"fmt"
5
+
"net/url"
6
+
)
7
+
8
+
type KnotSource struct {
9
+
Knot string
10
+
}
11
+
12
+
func (k KnotSource) Key() string {
13
+
return k.Knot
14
+
}
15
+
16
+
func (k KnotSource) Url(cursor int64, dev bool) (*url.URL, error) {
17
+
scheme := "wss"
18
+
if dev {
19
+
scheme = "ws"
20
+
}
21
+
22
+
u, err := url.Parse(scheme + "://" + k.Knot + "/events")
23
+
if err != nil {
24
+
return nil, err
25
+
}
26
+
27
+
if cursor != 0 {
28
+
query := url.Values{}
29
+
query.Add("cursor", fmt.Sprintf("%d", cursor))
30
+
u.RawQuery = query.Encode()
31
+
}
32
+
return u, nil
33
+
}
34
+
35
+
func NewKnotSource(knot string) KnotSource {
36
+
return KnotSource{
37
+
Knot: knot,
38
+
}
39
+
}
+39
eventconsumer/spindle.go
+39
eventconsumer/spindle.go
···
1
+
package eventconsumer
2
+
3
+
import (
4
+
"fmt"
5
+
"net/url"
6
+
)
7
+
8
+
type SpindleSource struct {
9
+
Spindle string
10
+
}
11
+
12
+
func (s SpindleSource) Key() string {
13
+
return s.Spindle
14
+
}
15
+
16
+
func (s SpindleSource) Url(cursor int64, dev bool) (*url.URL, error) {
17
+
scheme := "wss"
18
+
if dev {
19
+
scheme = "ws"
20
+
}
21
+
22
+
u, err := url.Parse(scheme + "://" + s.Spindle + "/events")
23
+
if err != nil {
24
+
return nil, err
25
+
}
26
+
27
+
if cursor != 0 {
28
+
query := url.Values{}
29
+
query.Add("cursor", fmt.Sprintf("%d", cursor))
30
+
u.RawQuery = query.Encode()
31
+
}
32
+
return u, nil
33
+
}
34
+
35
+
func NewSpindleSource(spindle string) SpindleSource {
36
+
return SpindleSource{
37
+
Spindle: spindle,
38
+
}
39
+
}
knotclient/cursor/memory.go
eventconsumer/cursor/memory.go
knotclient/cursor/memory.go
eventconsumer/cursor/memory.go
knotclient/cursor/redis.go
eventconsumer/cursor/redis.go
knotclient/cursor/redis.go
eventconsumer/cursor/redis.go
knotclient/cursor/sqlite.go
eventconsumer/cursor/sqlite.go
knotclient/cursor/sqlite.go
eventconsumer/cursor/sqlite.go
knotclient/cursor/store.go
eventconsumer/cursor/store.go
knotclient/cursor/store.go
eventconsumer/cursor/store.go
+25
-48
knotclient/events.go
eventconsumer/consumer.go
+25
-48
knotclient/events.go
eventconsumer/consumer.go
···
1
-
package knotclient
1
+
package eventconsumer
2
2
3
3
import (
4
4
"context"
5
5
"encoding/json"
6
-
"fmt"
7
6
"log/slog"
8
7
"math/rand"
9
8
"net/url"
10
9
"sync"
11
10
"time"
12
11
13
-
"tangled.sh/tangled.sh/core/knotclient/cursor"
12
+
"tangled.sh/tangled.sh/core/eventconsumer/cursor"
14
13
"tangled.sh/tangled.sh/core/log"
15
14
16
15
"github.com/gorilla/websocket"
17
16
)
18
17
19
-
type ProcessFunc func(ctx context.Context, source EventSource, message Message) error
18
+
type ProcessFunc func(ctx context.Context, source Source, message Message) error
20
19
21
20
type Message struct {
22
21
Rkey string
···
25
26
}
26
27
27
28
type ConsumerConfig struct {
28
-
Sources map[EventSource]struct{}
29
+
Sources map[Source]struct{}
29
30
ProcessFunc ProcessFunc
30
31
RetryInterval time.Duration
31
32
MaxRetryInterval time.Duration
···
39
40
40
41
func NewConsumerConfig() *ConsumerConfig {
41
42
return &ConsumerConfig{
42
-
Sources: make(map[EventSource]struct{}),
43
+
Sources: make(map[Source]struct{}),
43
44
}
44
45
}
45
46
46
-
type EventSource struct {
47
-
Knot string
47
+
type Source interface {
48
+
// url to start streaming events from
49
+
Url(cursor int64, dev bool) (*url.URL, error)
50
+
// cache key for cursor storage
51
+
Key() string
48
52
}
49
53
50
-
func NewEventSource(knot string) EventSource {
51
-
return EventSource{
52
-
Knot: knot,
53
-
}
54
-
}
55
-
56
-
type EventConsumer struct {
54
+
type Consumer struct {
57
55
wg sync.WaitGroup
58
56
dialer *websocket.Dialer
59
57
connMap sync.Map
···
63
67
cfg ConsumerConfig
64
68
}
65
69
66
-
func (e *EventConsumer) buildUrl(s EventSource, cursor int64) (*url.URL, error) {
67
-
scheme := "wss"
68
-
if e.cfg.Dev {
69
-
scheme = "ws"
70
-
}
71
-
72
-
u, err := url.Parse(scheme + "://" + s.Knot + "/events")
73
-
if err != nil {
74
-
return nil, err
75
-
}
76
-
77
-
if cursor != 0 {
78
-
query := url.Values{}
79
-
query.Add("cursor", fmt.Sprintf("%d", cursor))
80
-
u.RawQuery = query.Encode()
81
-
}
82
-
return u, nil
83
-
}
84
-
85
70
type job struct {
86
-
source EventSource
71
+
source Source
87
72
message []byte
88
73
}
89
74
90
-
func NewEventConsumer(cfg ConsumerConfig) *EventConsumer {
75
+
func NewConsumer(cfg ConsumerConfig) *Consumer {
91
76
if cfg.RetryInterval == 0 {
92
77
cfg.RetryInterval = 15 * time.Minute
93
78
}
···
82
105
cfg.MaxRetryInterval = 1 * time.Hour
83
106
}
84
107
if cfg.Logger == nil {
85
-
cfg.Logger = log.New("eventconsumer")
108
+
cfg.Logger = log.New("consumer")
86
109
}
87
110
if cfg.QueueSize == 0 {
88
111
cfg.QueueSize = 100
···
90
113
if cfg.CursorStore == nil {
91
114
cfg.CursorStore = &cursor.MemoryStore{}
92
115
}
93
-
return &EventConsumer{
116
+
return &Consumer{
94
117
cfg: cfg,
95
118
dialer: websocket.DefaultDialer,
96
119
jobQueue: make(chan job, cfg.QueueSize), // buffered job queue
···
99
122
}
100
123
}
101
124
102
-
func (c *EventConsumer) Start(ctx context.Context) {
125
+
func (c *Consumer) Start(ctx context.Context) {
103
126
c.cfg.Logger.Info("starting consumer", "config", c.cfg)
104
127
105
128
// start workers
···
115
138
}
116
139
}
117
140
118
-
func (c *EventConsumer) Stop() {
141
+
func (c *Consumer) Stop() {
119
142
c.connMap.Range(func(_, val any) bool {
120
143
if conn, ok := val.(*websocket.Conn); ok {
121
144
conn.Close()
···
126
149
close(c.jobQueue)
127
150
}
128
151
129
-
func (c *EventConsumer) AddSource(ctx context.Context, s EventSource) {
152
+
func (c *Consumer) AddSource(ctx context.Context, s Source) {
130
153
// we are already listening to this source
131
154
if _, ok := c.cfg.Sources[s]; ok {
132
155
c.logger.Info("source already present", "source", s)
···
140
163
c.cfgMu.Unlock()
141
164
}
142
165
143
-
func (c *EventConsumer) worker(ctx context.Context) {
166
+
func (c *Consumer) worker(ctx context.Context) {
144
167
defer c.wg.Done()
145
168
for {
146
169
select {
···
154
177
var msg Message
155
178
err := json.Unmarshal(j.message, &msg)
156
179
if err != nil {
157
-
c.logger.Error("error deserializing message", "source", j.source.Knot, "err", err)
180
+
c.logger.Error("error deserializing message", "source", j.source.Key(), "err", err)
158
181
return
159
182
}
160
183
161
184
// update cursor
162
-
c.cfg.CursorStore.Set(j.source.Knot, time.Now().UnixNano())
185
+
c.cfg.CursorStore.Set(j.source.Key(), time.Now().UnixNano())
163
186
164
187
if err := c.cfg.ProcessFunc(ctx, j.source, msg); err != nil {
165
188
c.logger.Error("error processing message", "source", j.source, "err", err)
···
168
191
}
169
192
}
170
193
171
-
func (c *EventConsumer) startConnectionLoop(ctx context.Context, source EventSource) {
194
+
func (c *Consumer) startConnectionLoop(ctx context.Context, source Source) {
172
195
defer c.wg.Done()
173
196
retryInterval := c.cfg.RetryInterval
174
197
for {
···
201
224
}
202
225
}
203
226
204
-
func (c *EventConsumer) runConnection(ctx context.Context, source EventSource) error {
227
+
func (c *Consumer) runConnection(ctx context.Context, source Source) error {
205
228
connCtx, cancel := context.WithTimeout(ctx, c.cfg.ConnectionTimeout)
206
229
defer cancel()
207
230
208
-
cursor := c.cfg.CursorStore.Get(source.Knot)
231
+
cursor := c.cfg.CursorStore.Get(source.Key())
209
232
210
-
u, err := c.buildUrl(source, cursor)
233
+
u, err := source.Url(cursor, c.cfg.Dev)
211
234
if err != nil {
212
235
return err
213
236
}
+2
-2
spindle/ingester.go
+2
-2
spindle/ingester.go
···
6
6
"fmt"
7
7
8
8
"tangled.sh/tangled.sh/core/api/tangled"
9
-
"tangled.sh/tangled.sh/core/knotclient"
9
+
"tangled.sh/tangled.sh/core/eventconsumer"
10
10
11
11
"github.com/bluesky-social/jetstream/pkg/models"
12
12
)
···
128
128
}
129
129
130
130
// add this knot to the event consumer
131
-
src := knotclient.NewEventSource(record.Knot)
131
+
src := eventconsumer.NewKnotSource(record.Knot)
132
132
s.ks.AddSource(context.Background(), src)
133
133
134
134
return nil
+8
-8
spindle/server.go
+8
-8
spindle/server.go
···
9
9
10
10
"github.com/go-chi/chi/v5"
11
11
"tangled.sh/tangled.sh/core/api/tangled"
12
+
"tangled.sh/tangled.sh/core/eventconsumer"
13
+
"tangled.sh/tangled.sh/core/eventconsumer/cursor"
12
14
"tangled.sh/tangled.sh/core/jetstream"
13
-
"tangled.sh/tangled.sh/core/knotclient"
14
-
"tangled.sh/tangled.sh/core/knotclient/cursor"
15
15
"tangled.sh/tangled.sh/core/log"
16
16
"tangled.sh/tangled.sh/core/notifier"
17
17
"tangled.sh/tangled.sh/core/rbac"
···
35
35
eng *engine.Engine
36
36
jq *queue.Queue
37
37
cfg *config.Config
38
-
ks *knotclient.EventConsumer
38
+
ks *eventconsumer.Consumer
39
39
}
40
40
41
41
func Run(ctx context.Context) error {
···
114
114
// for each incoming sh.tangled.pipeline, we execute
115
115
// spindle.processPipeline, which in turn enqueues the pipeline
116
116
// job in the above registered queue.
117
-
ccfg := knotclient.NewConsumerConfig()
117
+
ccfg := eventconsumer.NewConsumerConfig()
118
118
ccfg.Logger = logger
119
119
ccfg.Dev = cfg.Server.Dev
120
120
ccfg.ProcessFunc = spindle.processPipeline
···
125
125
}
126
126
for _, knot := range knownKnots {
127
127
logger.Info("adding source start", "knot", knot)
128
-
ccfg.Sources[knotclient.EventSource{knot}] = struct{}{}
128
+
ccfg.Sources[eventconsumer.NewKnotSource(knot)] = struct{}{}
129
129
}
130
-
spindle.ks = knotclient.NewEventConsumer(*ccfg)
130
+
spindle.ks = eventconsumer.NewConsumer(*ccfg)
131
131
132
132
go func() {
133
133
logger.Info("starting knot event consumer")
···
151
151
return mux
152
152
}
153
153
154
-
func (s *Spindle) processPipeline(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error {
154
+
func (s *Spindle) processPipeline(ctx context.Context, src eventconsumer.Source, msg eventconsumer.Message) error {
155
155
if msg.Nsid == tangled.PipelineNSID {
156
156
pipeline := tangled.Pipeline{}
157
157
err := json.Unmarshal(msg.EventJson, &pipeline)
···
179
179
}
180
180
181
181
pipelineId := models.PipelineId{
182
-
Knot: src.Knot,
182
+
Knot: src.Key(),
183
183
Rkey: msg.Rkey,
184
184
}
185
185
+24
-5
spindle/stream.go
+24
-5
spindle/stream.go
···
2
2
3
3
import (
4
4
"context"
5
+
"encoding/json"
5
6
"fmt"
6
7
"net/http"
7
8
"strconv"
···
207
206
}
208
207
209
208
func (s *Spindle) streamPipelines(conn *websocket.Conn, cursor *int64) error {
210
-
ops, err := s.db.GetEvents(*cursor)
209
+
events, err := s.db.GetEvents(*cursor)
211
210
if err != nil {
212
211
s.l.Debug("err", "err", err)
213
212
return err
214
213
}
215
-
s.l.Debug("ops", "ops", ops)
214
+
s.l.Debug("ops", "ops", events)
216
215
217
-
for _, op := range ops {
218
-
if err := conn.WriteJSON(op); err != nil {
216
+
for _, event := range events {
217
+
// first extract the inner json into a map
218
+
var eventJson map[string]any
219
+
err := json.Unmarshal([]byte(event.EventJson), &eventJson)
220
+
if err != nil {
221
+
s.l.Error("failed to unmarshal event", "err", err)
222
+
return err
223
+
}
224
+
225
+
jsonMsg, err := json.Marshal(map[string]any{
226
+
"rkey": event.Rkey,
227
+
"nsid": event.Nsid,
228
+
"event": eventJson,
229
+
})
230
+
if err != nil {
231
+
s.l.Error("failed to marshal record", "err", err)
232
+
return err
233
+
}
234
+
235
+
if err := conn.WriteMessage(websocket.TextMessage, jsonMsg); err != nil {
219
236
s.l.Debug("err", "err", err)
220
237
return err
221
238
}
222
-
*cursor = op.Created
239
+
*cursor = event.Created
223
240
}
224
241
225
242
return nil