forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
spindle: setup jetstream ingester
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.sh>
authored by
Anirudh Oppiliappan
and committed by
Tangled
504e391a
b0e291db
+158
-27
+3
-1
spindle/config/config.go
+3
-1
spindle/config/config.go
···
12
12
Hostname string `env:"HOSTNAME, required"`
13
13
JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"`
14
14
Dev bool `env:"DEV, default=false"`
15
+
Owner string `env:"OWNER, required"`
15
16
}
16
17
17
18
type Config struct {
18
-
Server Server `env:",prefix=SPINDLE_SERVER_"`
19
+
Server Server `env:",prefix=SPINDLE_SERVER_"`
20
+
Knots []string `env:"SPINDLE_SUBSCRIBED_KNOTS,required"`
19
21
}
20
22
21
23
func Load(ctx context.Context) (*Config, error) {
+87
spindle/ingester.go
+87
spindle/ingester.go
···
1
+
package spindle
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
8
+
"github.com/bluesky-social/jetstream/pkg/models"
9
+
"tangled.sh/tangled.sh/core/api/tangled"
10
+
)
11
+
12
+
type Ingester func(ctx context.Context, e *models.Event) error
13
+
14
+
func (s *Spindle) ingest() Ingester {
15
+
return func(ctx context.Context, e *models.Event) error {
16
+
var err error
17
+
defer func() {
18
+
eventTime := e.TimeUS
19
+
lastTimeUs := eventTime + 1
20
+
if err := s.db.SaveLastTimeUs(lastTimeUs); err != nil {
21
+
err = fmt.Errorf("(deferred) failed to save last time us: %w", err)
22
+
}
23
+
}()
24
+
25
+
if e.Kind != models.EventKindCommit {
26
+
return nil
27
+
}
28
+
29
+
switch e.Commit.Collection {
30
+
case tangled.SpindleMemberNSID:
31
+
s.ingestMember(ctx, e)
32
+
}
33
+
34
+
return err
35
+
}
36
+
}
37
+
38
+
func (s *Spindle) ingestMember(_ context.Context, e *models.Event) error {
39
+
did := e.Did
40
+
var err error
41
+
42
+
l := s.l.With("component", "ingester", "record", tangled.SpindleMemberNSID)
43
+
44
+
switch e.Commit.Operation {
45
+
case models.CommitOperationCreate, models.CommitOperationUpdate:
46
+
raw := e.Commit.Record
47
+
record := tangled.SpindleMember{}
48
+
err = json.Unmarshal(raw, &record)
49
+
if err != nil {
50
+
l.Error("invalid record", "error", err)
51
+
return err
52
+
}
53
+
54
+
domain := s.cfg.Server.Hostname
55
+
if s.cfg.Server.Dev {
56
+
domain = s.cfg.Server.ListenAddr
57
+
}
58
+
recordInstance := *record.Instance
59
+
60
+
if recordInstance != domain {
61
+
l.Error("domain mismatch", "domain", recordInstance, "expected", domain)
62
+
return fmt.Errorf("domain mismatch: %s != %s", *record.Instance, domain)
63
+
}
64
+
65
+
ok, err := s.e.E.Enforce(did, rbacDomain, rbacDomain, "server:invite")
66
+
if err != nil || !ok {
67
+
l.Error("failed to add member", "did", did)
68
+
return fmt.Errorf("failed to enforce permissions: %w", err)
69
+
}
70
+
71
+
if err := s.e.AddMember(rbacDomain, record.Subject); err != nil {
72
+
l.Error("failed to add member", "error", err)
73
+
return fmt.Errorf("failed to add member: %w", err)
74
+
}
75
+
l.Info("added member from firehose", "member", record.Subject)
76
+
77
+
if err := s.db.AddDid(did); err != nil {
78
+
l.Error("failed to add did", "error", err)
79
+
return fmt.Errorf("failed to add did: %w", err)
80
+
}
81
+
s.jc.AddDid(did)
82
+
83
+
return nil
84
+
85
+
}
86
+
return nil
87
+
}
+68
-26
spindle/server.go
+68
-26
spindle/server.go
···
22
22
"tangled.sh/tangled.sh/core/spindle/queue"
23
23
)
24
24
25
+
const (
26
+
rbacDomain = "thisserver"
27
+
)
28
+
25
29
type Spindle struct {
26
30
jc *jetstream.JetstreamClient
27
31
db *db.DB
···
34
30
n *notifier.Notifier
35
31
eng *engine.Engine
36
32
jq *queue.Queue
33
+
cfg *config.Config
37
34
}
38
35
39
36
func Run(ctx context.Context) error {
37
+
logger := log.FromContext(ctx)
38
+
40
39
cfg, err := config.Load(ctx)
41
40
if err != nil {
42
41
return fmt.Errorf("failed to load config: %w", err)
···
54
47
if err != nil {
55
48
return fmt.Errorf("failed to setup rbac enforcer: %w", err)
56
49
}
57
-
58
-
logger := log.FromContext(ctx)
59
-
60
-
collections := []string{tangled.SpindleMemberNSID}
61
-
jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, logger, d, true, false)
62
-
if err != nil {
63
-
return fmt.Errorf("failed to setup jetstream client: %w", err)
64
-
}
50
+
e.E.EnableAutoSave(true)
65
51
66
52
n := notifier.New()
53
+
67
54
eng, err := engine.New(ctx, d, &n)
68
55
if err != nil {
69
56
return err
···
65
64
66
65
jq := queue.NewQueue(100, 2)
67
66
68
-
// starts a job queue runner in the background
69
-
jq.Start()
70
-
defer jq.Stop()
67
+
collections := []string{tangled.SpindleMemberNSID}
68
+
jc, err := jetstream.NewJetstreamClient(cfg.Server.JetstreamEndpoint, "spindle", collections, nil, logger, d, false, false)
69
+
if err != nil {
70
+
return fmt.Errorf("failed to setup jetstream client: %w", err)
71
+
}
71
72
72
73
spindle := Spindle{
73
74
jc: jc,
···
79
76
n: &n,
80
77
eng: eng,
81
78
jq: jq,
79
+
cfg: cfg,
80
+
}
81
+
82
+
err = e.AddDomain(rbacDomain)
83
+
if err != nil {
84
+
return fmt.Errorf("failed to set rbac domain: %w", err)
85
+
}
86
+
err = spindle.configureOwner()
87
+
if err != nil {
88
+
return err
89
+
}
90
+
logger.Info("owner set", "did", cfg.Server.Owner)
91
+
92
+
// starts a job queue runner in the background
93
+
jq.Start()
94
+
defer jq.Stop()
95
+
96
+
cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath)
97
+
if err != nil {
98
+
return fmt.Errorf("failed to setup sqlite3 cursor store: %w", err)
99
+
}
100
+
101
+
err = jc.StartJetstream(ctx, spindle.ingest())
102
+
if err != nil {
103
+
return fmt.Errorf("failed to start jetstream consumer: %w", err)
82
104
}
83
105
84
106
// for each incoming sh.tangled.pipeline, we execute
85
107
// spindle.processPipeline, which in turn enqueues the pipeline
86
108
// job in the above registered queue.
87
-
cursorStore, err := cursor.NewSQLiteStore(cfg.Server.DBPath)
88
-
if err != nil {
89
-
return fmt.Errorf("failed to setup sqlite3 cursor store: %w", err)
109
+
110
+
ccfg := knotclient.NewConsumerConfig()
111
+
ccfg.Logger = logger
112
+
ccfg.Dev = cfg.Server.Dev
113
+
ccfg.ProcessFunc = spindle.processPipeline
114
+
ccfg.CursorStore = cursorStore
115
+
for _, knot := range spindle.cfg.Knots {
116
+
kes := knotclient.NewEventSource(knot)
117
+
ccfg.AddEventSource(kes)
90
118
}
119
+
ec := knotclient.NewEventConsumer(*ccfg)
120
+
91
121
go func() {
92
-
logger.Info("starting event consumer")
93
-
knotEventSource := knotclient.NewEventSource("localhost:6000")
94
-
95
-
ccfg := knotclient.NewConsumerConfig()
96
-
ccfg.Logger = logger
97
-
ccfg.Dev = cfg.Server.Dev
98
-
ccfg.ProcessFunc = spindle.processPipeline
99
-
ccfg.CursorStore = cursorStore
100
-
ccfg.AddEventSource(knotEventSource)
101
-
102
-
ec := knotclient.NewEventConsumer(*ccfg)
103
-
122
+
logger.Info("starting knot event consumer", "knots", spindle.cfg.Knots)
104
123
ec.Start(ctx)
105
124
}()
106
125
···
182
157
}
183
158
}
184
159
160
+
return nil
161
+
}
162
+
163
+
func (s *Spindle) configureOwner() error {
164
+
cfgOwner := s.cfg.Server.Owner
165
+
serverOwner, err := s.e.GetUserByRole("server:owner", rbacDomain)
166
+
if err != nil {
167
+
return fmt.Errorf("failed to fetch server:owner: %w", err)
168
+
}
169
+
170
+
if len(serverOwner) == 0 {
171
+
s.e.AddOwner(rbacDomain, cfgOwner)
172
+
} else {
173
+
if serverOwner[0] != cfgOwner {
174
+
return fmt.Errorf("server owner mismatch: %s != %s", cfgOwner, serverOwner[0])
175
+
}
176
+
}
185
177
return nil
186
178
}