forked from
tangled.org/core
Mirror of @tangled.org/core. Running on a Raspberry Pi Zero 2 (Please be gentle).
spindle/{db,engine}: rework db to use rkey
Signed-off-by: Anirudh Oppiliappan <anirudh@tangled.sh>
+86
-63
+1
spindle/config/config.go
+1
spindle/config/config.go
···
11
11
DBPath string `env:"DB_PATH, default=spindle.db"`
12
12
Hostname string `env:"HOSTNAME, required"`
13
13
JetstreamEndpoint string `env:"JETSTREAM_ENDPOINT, default=wss://jetstream1.us-west.bsky.network/subscribe"`
14
+
Dev bool `env:"DEV, default=false"`
14
15
}
15
16
16
17
type Config struct {
+4
-3
spindle/db/db.go
+4
-3
spindle/db/db.go
···
30
30
did text primary key
31
31
);
32
32
33
-
create table if not exists pipelines (
34
-
at_uri text not null,
33
+
create table if not exists pipeline_status (
34
+
rkey text not null,
35
+
pipeline text not null,
35
36
status text not null,
36
37
37
38
-- only set if status is 'failed'
···
43
42
updated_at timestamp not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
44
43
finished_at timestamp,
45
44
46
-
primary key (at_uri)
45
+
primary key (rkey)
47
46
);
48
47
`)
49
48
if err != nil {
+48
-42
spindle/db/pipelines.go
+48
-42
spindle/db/pipelines.go
···
8
8
"tangled.sh/tangled.sh/core/knotserver/notifier"
9
9
)
10
10
11
-
type PipelineStatus string
11
+
type PipelineRunStatus string
12
12
13
13
var (
14
-
PipelinePending PipelineStatus = "pending"
15
-
PipelineRunning PipelineStatus = "running"
16
-
PipelineFailed PipelineStatus = "failed"
17
-
PipelineTimeout PipelineStatus = "timeout"
18
-
PipelineCancelled PipelineStatus = "cancelled"
19
-
PipelineSuccess PipelineStatus = "success"
14
+
PipelinePending PipelineRunStatus = "pending"
15
+
PipelineRunning PipelineRunStatus = "running"
16
+
PipelineFailed PipelineRunStatus = "failed"
17
+
PipelineTimeout PipelineRunStatus = "timeout"
18
+
PipelineCancelled PipelineRunStatus = "cancelled"
19
+
PipelineSuccess PipelineRunStatus = "success"
20
20
)
21
21
22
-
type Pipeline struct {
23
-
Rkey string `json:"rkey"`
24
-
Knot string `json:"knot"`
25
-
Status PipelineStatus `json:"status"`
22
+
type PipelineStatus struct {
23
+
Rkey string `json:"rkey"`
24
+
Pipeline string `json:"pipeline"`
25
+
Status PipelineRunStatus `json:"status"`
26
26
27
27
// only if Failed
28
28
Error string `json:"error"`
···
33
33
FinishedAt time.Time `json:"finished_at"`
34
34
}
35
35
36
-
func (p Pipeline) AsRecord() *tangled.PipelineStatus {
36
+
func (p PipelineStatus) AsRecord() *tangled.PipelineStatus {
37
37
exitCode64 := int64(p.ExitCode)
38
38
finishedAt := p.FinishedAt.String()
39
39
40
40
return &tangled.PipelineStatus{
41
-
Pipeline: fmt.Sprintf("at://%s/%s", p.Knot, p.Rkey),
42
-
Status: string(p.Status),
41
+
LexiconTypeID: tangled.PipelineStatusNSID,
42
+
Pipeline: p.Pipeline,
43
+
Status: string(p.Status),
43
44
44
45
ExitCode: &exitCode64,
45
46
Error: &p.Error,
···
55
54
return fmt.Sprintf("at://%s/did:web:%s/%s", tangled.PipelineStatusNSID, knot, rkey)
56
55
}
57
56
58
-
func (db *DB) CreatePipeline(rkey, knot string, n *notifier.Notifier) error {
57
+
func (db *DB) CreatePipeline(rkey, pipeline string, n *notifier.Notifier) error {
59
58
_, err := db.Exec(`
60
-
insert into pipelines (at_uri, status)
61
-
values (?, ?)
62
-
`, pipelineAtUri(rkey, knot), PipelinePending)
59
+
insert into pipeline_status (rkey, status, pipeline)
60
+
values (?, ?, ?)
61
+
`, rkey, PipelinePending, pipeline)
63
62
64
63
if err != nil {
65
64
return err
···
68
67
return nil
69
68
}
70
69
71
-
func (db *DB) MarkPipelineRunning(rkey, knot string, n *notifier.Notifier) error {
70
+
func (db *DB) MarkPipelineRunning(rkey string, n *notifier.Notifier) error {
72
71
_, err := db.Exec(`
73
-
update pipelines
72
+
update pipeline_status
74
73
set status = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
75
-
where at_uri = ?
76
-
`, PipelineRunning, pipelineAtUri(rkey, knot))
74
+
where rkey = ?
75
+
`, PipelineRunning, rkey)
77
76
78
77
if err != nil {
79
78
return err
···
82
81
return nil
83
82
}
84
83
85
-
func (db *DB) MarkPipelineFailed(rkey, knot string, exitCode int, errorMsg string, n *notifier.Notifier) error {
84
+
func (db *DB) MarkPipelineFailed(rkey string, exitCode int, errorMsg string, n *notifier.Notifier) error {
86
85
_, err := db.Exec(`
87
-
update pipelines
86
+
update pipeline_status
88
87
set status = ?,
89
88
exit_code = ?,
90
89
error = ?,
91
90
updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now'),
92
91
finished_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
93
-
where at_uri = ?
94
-
`, PipelineFailed, exitCode, errorMsg, pipelineAtUri(rkey, knot))
92
+
where rkey = ?
93
+
`, PipelineFailed, exitCode, errorMsg, rkey)
95
94
if err != nil {
96
95
return err
97
96
}
···
99
98
return nil
100
99
}
101
100
102
-
func (db *DB) MarkPipelineTimeout(rkey, knot string, n *notifier.Notifier) error {
101
+
func (db *DB) MarkPipelineTimeout(rkey string, n *notifier.Notifier) error {
103
102
_, err := db.Exec(`
104
-
update pipelines
103
+
update pipeline_status
105
104
set status = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
106
-
where at_uri = ?
107
-
`, PipelineTimeout, pipelineAtUri(rkey, knot))
105
+
where rkey = ?
106
+
`, PipelineTimeout, rkey)
108
107
if err != nil {
109
108
return err
110
109
}
···
112
111
return nil
113
112
}
114
113
115
-
func (db *DB) MarkPipelineSuccess(rkey, knot string, n *notifier.Notifier) error {
114
+
func (db *DB) MarkPipelineSuccess(rkey string, n *notifier.Notifier) error {
116
115
_, err := db.Exec(`
117
-
update pipelines
116
+
update pipeline_status
118
117
set status = ?, updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now'),
119
118
finished_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')
120
-
where at_uri = ?
121
-
`, PipelineSuccess, pipelineAtUri(rkey, knot))
119
+
where rkey = ?
120
+
`, PipelineSuccess, rkey)
122
121
123
122
if err != nil {
124
123
return err
···
127
126
return nil
128
127
}
129
128
130
-
func (db *DB) GetPipeline(rkey, knot string) (Pipeline, error) {
131
-
var p Pipeline
129
+
func (db *DB) GetPipelineStatus(rkey string) (PipelineStatus, error) {
130
+
var p PipelineStatus
132
131
err := db.QueryRow(`
133
132
select rkey, status, error, exit_code, started_at, updated_at, finished_at
134
133
from pipelines
135
-
where at_uri = ?
136
-
`, pipelineAtUri(rkey, knot)).Scan(&p.Rkey, &p.Status, &p.Error, &p.ExitCode, &p.StartedAt, &p.UpdatedAt, &p.FinishedAt)
134
+
where rkey = ?
135
+
`, rkey).Scan(&p.Rkey, &p.Status, &p.Error, &p.ExitCode, &p.StartedAt, &p.UpdatedAt, &p.FinishedAt)
137
136
return p, err
138
137
}
139
138
140
-
func (db *DB) GetPipelines(cursor string) ([]Pipeline, error) {
139
+
func (db *DB) GetPipelineStatusAsRecords(cursor string) ([]PipelineStatus, error) {
141
140
whereClause := ""
142
141
args := []any{}
143
142
if cursor != "" {
···
147
146
148
147
query := fmt.Sprintf(`
149
148
select rkey, status, error, exit_code, started_at, updated_at, finished_at
150
-
from pipelines
149
+
from pipeline_status
151
150
%s
152
151
order by rkey asc
153
152
limit 100
···
159
158
}
160
159
defer rows.Close()
161
160
162
-
var pipelines []Pipeline
161
+
var pipelines []PipelineStatus
163
162
for rows.Next() {
164
-
var p Pipeline
163
+
var p PipelineStatus
165
164
rows.Scan(&p.Rkey, &p.Status, &p.Error, &p.ExitCode, &p.StartedAt, &p.UpdatedAt, &p.FinishedAt)
166
165
pipelines = append(pipelines, p)
167
166
}
168
167
169
168
if err := rows.Err(); err != nil {
170
169
return nil, err
170
+
}
171
+
172
+
records := []*tangled.PipelineStatus{}
173
+
for _, p := range pipelines {
174
+
records = append(records, p.AsRecord())
171
175
}
172
176
173
177
return pipelines, nil
+2
-2
spindle/engine/engine.go
+2
-2
spindle/engine/engine.go
···
47
47
48
48
// SetupPipeline sets up a new network for the pipeline, and possibly volumes etc.
49
49
// in the future. In here also goes other setup steps.
50
-
func (e *Engine) SetupPipeline(ctx context.Context, pipeline *tangled.Pipeline, id string) error {
50
+
func (e *Engine) SetupPipeline(ctx context.Context, pipeline *tangled.Pipeline, atUri, id string) error {
51
51
e.l.Info("setting up pipeline", "pipeline", id)
52
52
53
53
_, err := e.docker.VolumeCreate(ctx, volume.CreateOptions{
···
73
73
return err
74
74
}
75
75
76
-
err = e.db.CreatePipeline(id, e.n)
76
+
err = e.db.CreatePipeline(id, atUri, e.n)
77
77
return err
78
78
}
79
79
+19
-14
spindle/server.go
+19
-14
spindle/server.go
···
70
70
go func() {
71
71
logger.Info("starting event consumer")
72
72
knotEventSource := knotclient.NewEventSource("localhost:5555")
73
-
ccfg := knotclient.ConsumerConfig{
74
-
Logger: logger,
75
-
ProcessFunc: spindle.exec,
76
-
}
73
+
74
+
ccfg := knotclient.NewConsumerConfig()
75
+
ccfg.Logger = logger
76
+
ccfg.Dev = cfg.Server.Dev
77
+
ccfg.ProcessFunc = spindle.exec
77
78
ccfg.AddEventSource(knotEventSource)
78
79
79
-
ec := knotclient.NewEventConsumer(ccfg)
80
+
ec := knotclient.NewEventConsumer(*ccfg)
80
81
81
82
ec.Start(ctx)
82
83
}()
···
96
95
}
97
96
98
97
func (s *Spindle) exec(ctx context.Context, src knotclient.EventSource, msg knotclient.Message) error {
99
-
pipeline := tangled.Pipeline{}
100
-
err := json.Unmarshal(msg.EventJson, &pipeline)
101
-
if err != nil {
102
-
fmt.Println("error unmarshalling", err)
103
-
return err
104
-
}
105
-
106
98
if msg.Nsid == tangled.PipelineNSID {
107
-
err = s.eng.SetupPipeline(ctx, &pipeline, msg.Rkey)
99
+
pipeline := tangled.Pipeline{}
100
+
err := json.Unmarshal(msg.EventJson, &pipeline)
101
+
if err != nil {
102
+
fmt.Println("error unmarshalling", err)
103
+
return err
104
+
}
105
+
106
+
// this is a "fake" at uri for now
107
+
pipelineAtUri := fmt.Sprintf("at://%s/did:web:%s/%s", tangled.PipelineNSID, pipeline.TriggerMetadata.Repo.Knot, msg.Rkey)
108
+
109
+
rkey := TID()
110
+
err = s.eng.SetupPipeline(ctx, &pipeline, pipelineAtUri, rkey)
108
111
if err != nil {
109
112
return err
110
113
}
111
-
err = s.eng.StartWorkflows(ctx, &pipeline, msg.Rkey)
114
+
err = s.eng.StartWorkflows(ctx, &pipeline, rkey)
112
115
if err != nil {
113
116
return err
114
117
}
+3
-2
spindle/stream.go
+3
-2
spindle/stream.go
···
4
4
"net/http"
5
5
"time"
6
6
7
+
"context"
8
+
7
9
"github.com/gorilla/websocket"
8
-
"golang.org/x/net/context"
9
10
)
10
11
11
12
var upgrader = websocket.Upgrader{
···
75
74
}
76
75
77
76
func (s *Spindle) streamPipelines(conn *websocket.Conn, cursor *string) error {
78
-
ops, err := s.db.GetPipelines(*cursor)
77
+
ops, err := s.db.GetPipelineStatusAsRecords(*cursor)
79
78
if err != nil {
80
79
s.l.Debug("err", "err", err)
81
80
return err