forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
appview,knotserver,spindle: rework jetstream
do not return errors from ingesters, this causes the read loop to be
killed.
Signed-off-by: oppiliappan <me@oppi.li>
authored by
oppiliappan
and committed by
Tangled
c8d76b33
bb4ac7ec
+71
-59
+2
-2
appview/ingester.go
+2
-2
appview/ingester.go
+6
-4
jetstream/jetstream.go
+6
-4
jetstream/jetstream.go
···
68
68
type processor func(context.Context, *models.Event) error
69
69
70
70
func (j *JetstreamClient) withDidFilter(processFunc processor) processor {
71
-
// empty filter => all dids allowed
72
-
if len(j.wantedDids) == 0 {
73
-
return processFunc
74
-
}
75
71
// since this closure references j.WantedDids; it should auto-update
76
72
// existing instances of the closure when j.WantedDids is mutated
77
73
return func(ctx context.Context, evt *models.Event) error {
74
+
75
+
// empty filter => all dids allowed
76
+
if len(j.wantedDids) == 0 {
77
+
return processFunc(ctx, evt)
78
+
}
79
+
78
80
if _, ok := j.wantedDids[evt.Did]; ok {
79
81
return processFunc(ctx, evt)
80
82
} else {
+5
-5
knotserver/handler.go
+5
-5
knotserver/handler.go
···
52
52
return nil, fmt.Errorf("failed to setup enforcer: %w", err)
53
53
}
54
54
55
-
err = h.jc.StartJetstream(ctx, h.processMessages)
56
-
if err != nil {
57
-
return nil, fmt.Errorf("failed to start jetstream: %w", err)
58
-
}
59
-
60
55
// Check if the knot knows about any Dids;
61
56
// if it does, it is already initialized and we can repopulate the
62
57
// Jetstream subscriptions.
···
66
71
for _, d := range dids {
67
72
h.jc.AddDid(d)
68
73
}
74
+
}
75
+
76
+
err = h.jc.StartJetstream(ctx, h.processMessages)
77
+
if err != nil {
78
+
return nil, fmt.Errorf("failed to start jetstream: %w", err)
69
79
}
70
80
71
81
r.Get("/", h.Index)
+47
-43
knotserver/ingester.go
+47
-43
knotserver/ingester.go
···
25
25
"tangled.sh/tangled.sh/core/workflow"
26
26
)
27
27
28
-
func (h *Handle) processPublicKey(ctx context.Context, did string, record tangled.PublicKey) error {
28
+
func (h *Handle) processPublicKey(ctx context.Context, event *models.Event) error {
29
29
l := log.FromContext(ctx)
30
+
raw := json.RawMessage(event.Commit.Record)
31
+
did := event.Did
32
+
33
+
var record tangled.PublicKey
34
+
if err := json.Unmarshal(raw, &record); err != nil {
35
+
return fmt.Errorf("failed to unmarshal record: %w", err)
36
+
}
37
+
30
38
pk := db.PublicKey{
31
39
Did: did,
32
40
PublicKey: record,
···
47
39
return nil
48
40
}
49
41
50
-
func (h *Handle) processKnotMember(ctx context.Context, did string, record tangled.KnotMember) error {
42
+
func (h *Handle) processKnotMember(ctx context.Context, event *models.Event) error {
51
43
l := log.FromContext(ctx)
44
+
raw := json.RawMessage(event.Commit.Record)
45
+
did := event.Did
46
+
47
+
var record tangled.KnotMember
48
+
if err := json.Unmarshal(raw, &record); err != nil {
49
+
return fmt.Errorf("failed to unmarshal record: %w", err)
50
+
}
52
51
53
52
if record.Domain != h.c.Server.Hostname {
54
53
l.Error("domain mismatch", "domain", record.Domain, "expected", h.c.Server.Hostname)
···
87
72
return nil
88
73
}
89
74
90
-
func (h *Handle) processPull(ctx context.Context, did string, record tangled.RepoPull) error {
75
+
func (h *Handle) processPull(ctx context.Context, event *models.Event) error {
76
+
raw := json.RawMessage(event.Commit.Record)
77
+
did := event.Did
78
+
79
+
var record tangled.RepoPull
80
+
if err := json.Unmarshal(raw, &record); err != nil {
81
+
return fmt.Errorf("failed to unmarshal record: %w", err)
82
+
}
83
+
91
84
l := log.FromContext(ctx)
92
85
l = l.With("handler", "processPull")
93
86
l = l.With("did", did)
···
223
200
return nil
224
201
}
225
202
226
-
event := db.Event{
203
+
ev := db.Event{
227
204
Rkey: TID(),
228
205
Nsid: tangled.PipelineNSID,
229
206
EventJson: string(eventJson),
230
207
}
231
208
232
-
return h.db.InsertEvent(event, h.n)
209
+
return h.db.InsertEvent(ev, h.n)
233
210
}
234
211
235
212
// duplicated from add collaborator
236
-
func (h *Handle) processCollaborator(ctx context.Context, did string, record tangled.RepoCollaborator) error {
213
+
func (h *Handle) processCollaborator(ctx context.Context, event *models.Event) error {
214
+
raw := json.RawMessage(event.Commit.Record)
215
+
did := event.Did
216
+
217
+
var record tangled.RepoCollaborator
218
+
if err := json.Unmarshal(raw, &record); err != nil {
219
+
return fmt.Errorf("failed to unmarshal record: %w", err)
220
+
}
221
+
237
222
repoAt, err := syntax.ParseATURI(record.Repo)
238
223
if err != nil {
239
224
return err
···
274
243
didSlashRepo, _ := securejoin.SecureJoin(owner.DID.String(), repo.Name)
275
244
276
245
// check perms for this user
277
-
if ok, err := h.e.IsCollaboratorInviteAllowed(owner.DID.String(), rbac.ThisServer, didSlashRepo); !ok || err != nil {
246
+
if ok, err := h.e.IsCollaboratorInviteAllowed(did, rbac.ThisServer, didSlashRepo); !ok || err != nil {
278
247
return fmt.Errorf("insufficient permissions: %w", err)
279
248
}
280
249
···
334
303
}
335
304
336
305
func (h *Handle) processMessages(ctx context.Context, event *models.Event) error {
337
-
did := event.Did
338
306
if event.Kind != models.EventKindCommit {
339
307
return nil
340
308
}
···
347
317
}
348
318
}()
349
319
350
-
raw := json.RawMessage(event.Commit.Record)
351
-
352
320
switch event.Commit.Collection {
353
321
case tangled.PublicKeyNSID:
354
-
var record tangled.PublicKey
355
-
if err := json.Unmarshal(raw, &record); err != nil {
356
-
return fmt.Errorf("failed to unmarshal record: %w", err)
357
-
}
358
-
if err := h.processPublicKey(ctx, did, record); err != nil {
359
-
return fmt.Errorf("failed to process public key: %w", err)
360
-
}
361
-
322
+
err = h.processPublicKey(ctx, event)
362
323
case tangled.KnotMemberNSID:
363
-
var record tangled.KnotMember
364
-
if err := json.Unmarshal(raw, &record); err != nil {
365
-
return fmt.Errorf("failed to unmarshal record: %w", err)
366
-
}
367
-
if err := h.processKnotMember(ctx, did, record); err != nil {
368
-
return fmt.Errorf("failed to process knot member: %w", err)
369
-
}
370
-
324
+
err = h.processKnotMember(ctx, event)
371
325
case tangled.RepoPullNSID:
372
-
var record tangled.RepoPull
373
-
if err := json.Unmarshal(raw, &record); err != nil {
374
-
return fmt.Errorf("failed to unmarshal record: %w", err)
375
-
}
376
-
if err := h.processPull(ctx, did, record); err != nil {
377
-
return fmt.Errorf("failed to process knot member: %w", err)
378
-
}
379
-
326
+
err = h.processPull(ctx, event)
380
327
case tangled.RepoCollaboratorNSID:
381
-
var record tangled.RepoCollaborator
382
-
if err := json.Unmarshal(raw, &record); err != nil {
383
-
return fmt.Errorf("failed to unmarshal record: %w", err)
384
-
}
385
-
if err := h.processCollaborator(ctx, did, record); err != nil {
386
-
return fmt.Errorf("failed to process knot member: %w", err)
387
-
}
388
-
328
+
err = h.processCollaborator(ctx, event)
389
329
}
390
330
391
-
return err
331
+
if err != nil {
332
+
h.l.Debug("failed to process event", "nsid", event.Commit.Collection, "err", err)
333
+
}
334
+
335
+
return nil
392
336
}
+3
-1
log/log.go
+3
-1
log/log.go
···
9
9
// NewHandler sets up a new slog.Handler with the service name
10
10
// as an attribute
11
11
func NewHandler(name string) slog.Handler {
12
-
handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{})
12
+
handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
13
+
Level: slog.LevelDebug,
14
+
})
13
15
14
16
var attrs []slog.Attr
15
17
attrs = append(attrs, slog.Attr{Key: "service", Value: slog.StringValue(name)})
+8
-4
spindle/ingester.go
+8
-4
spindle/ingester.go
···
40
40
41
41
switch e.Commit.Collection {
42
42
case tangled.SpindleMemberNSID:
43
-
s.ingestMember(ctx, e)
43
+
err = s.ingestMember(ctx, e)
44
44
case tangled.RepoNSID:
45
-
s.ingestRepo(ctx, e)
45
+
err = s.ingestRepo(ctx, e)
46
46
case tangled.RepoCollaboratorNSID:
47
-
s.ingestCollaborator(ctx, e)
47
+
err = s.ingestCollaborator(ctx, e)
48
48
}
49
49
50
-
return err
50
+
if err != nil {
51
+
s.l.Debug("failed to process message", "nsid", e.Commit.Collection, "err", err)
52
+
}
53
+
54
+
return nil
51
55
}
52
56
}
53
57