this repo has no description
MST implementation with support for operation inversion (#914)
This is fairly similar to `indigo/mst`, but written from scratch in a
more go-idiomatic style, with the intent of supporting record op
inversion (for the "inductive firehose" proposal).
Goal is to have something that passes full interop tests with TypeScript
codebase, and can validate commits on the firehose. Would take more perf
work after that, and probably review, to get this in a state where it
could be in the hot path in prod.
+4223
+73
atproto/repo/car.go
+73
atproto/repo/car.go
···
1
+
package repo
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"io"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/repo/mst"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
12
+
"github.com/ipfs/go-datastore"
13
+
blockstore "github.com/ipfs/go-ipfs-blockstore"
14
+
"github.com/ipld/go-car"
15
+
)
16
+
17
+
func LoadFromCAR(ctx context.Context, r io.Reader) (*Repo, error) {
18
+
19
+
bs := blockstore.NewBlockstore(datastore.NewMapDatastore())
20
+
21
+
cr, err := car.NewCarReader(r)
22
+
if err != nil {
23
+
return nil, err
24
+
}
25
+
26
+
if cr.Header.Version != 1 {
27
+
return nil, fmt.Errorf("unsupported CAR file version: %d", cr.Header.Version)
28
+
}
29
+
if len(cr.Header.Roots) < 1 {
30
+
return nil, fmt.Errorf("CAR file missing root CID")
31
+
}
32
+
commitCID := cr.Header.Roots[0]
33
+
34
+
for {
35
+
blk, err := cr.Next()
36
+
if err != nil {
37
+
if err == io.EOF {
38
+
break
39
+
}
40
+
return nil, err
41
+
}
42
+
43
+
if err := bs.Put(ctx, blk); err != nil {
44
+
return nil, err
45
+
}
46
+
}
47
+
48
+
commitBlock, err := bs.Get(ctx, commitCID)
49
+
if err != nil {
50
+
return nil, fmt.Errorf("reading commit block from CAR file: %w", err)
51
+
}
52
+
53
+
var commit Commit
54
+
if err := commit.UnmarshalCBOR(bytes.NewReader(commitBlock.RawData())); err != nil {
55
+
return nil, fmt.Errorf("parsing commit block from CAR file: %w", err)
56
+
}
57
+
if err := commit.VerifyStructure(); err != nil {
58
+
return nil, fmt.Errorf("parsing commit block from CAR file: %w", err)
59
+
}
60
+
61
+
tree, err := mst.LoadTreeFromStore(ctx, bs, commit.Data)
62
+
if err != nil {
63
+
return nil, fmt.Errorf("reading MST from CAR file: %w", err)
64
+
}
65
+
repo := Repo{
66
+
DID: syntax.DID(commit.DID), // VerifyStructure() verified syntax
67
+
Clock: syntax.NewTIDClock(0), // TODO: initialize with commit.Rev
68
+
Commit: &commit,
69
+
MST: *tree,
70
+
RecordStore: bs, // TODO: put just records in a smaller blockstore?
71
+
}
72
+
return &repo, nil
73
+
}
+332
atproto/repo/cbor_gen.go
+332
atproto/repo/cbor_gen.go
···
1
+
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
2
+
3
+
package repo
4
+
5
+
import (
6
+
"fmt"
7
+
"io"
8
+
"math"
9
+
"sort"
10
+
11
+
cid "github.com/ipfs/go-cid"
12
+
cbg "github.com/whyrusleeping/cbor-gen"
13
+
xerrors "golang.org/x/xerrors"
14
+
)
15
+
16
+
var _ = xerrors.Errorf
17
+
var _ = cid.Undef
18
+
var _ = math.E
19
+
var _ = sort.Sort
20
+
21
+
func (t *Commit) MarshalCBOR(w io.Writer) error {
22
+
if t == nil {
23
+
_, err := w.Write(cbg.CborNull)
24
+
return err
25
+
}
26
+
27
+
cw := cbg.NewCborWriter(w)
28
+
fieldCount := 6
29
+
30
+
if t.Rev == "" {
31
+
fieldCount--
32
+
}
33
+
34
+
if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil {
35
+
return err
36
+
}
37
+
38
+
// t.DID (string) (string)
39
+
if len("did") > 1000000 {
40
+
return xerrors.Errorf("Value in field \"did\" was too long")
41
+
}
42
+
43
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil {
44
+
return err
45
+
}
46
+
if _, err := cw.WriteString(string("did")); err != nil {
47
+
return err
48
+
}
49
+
50
+
if len(t.DID) > 1000000 {
51
+
return xerrors.Errorf("Value in field t.DID was too long")
52
+
}
53
+
54
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.DID))); err != nil {
55
+
return err
56
+
}
57
+
if _, err := cw.WriteString(string(t.DID)); err != nil {
58
+
return err
59
+
}
60
+
61
+
// t.Rev (string) (string)
62
+
if t.Rev != "" {
63
+
64
+
if len("rev") > 1000000 {
65
+
return xerrors.Errorf("Value in field \"rev\" was too long")
66
+
}
67
+
68
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("rev"))); err != nil {
69
+
return err
70
+
}
71
+
if _, err := cw.WriteString(string("rev")); err != nil {
72
+
return err
73
+
}
74
+
75
+
if len(t.Rev) > 1000000 {
76
+
return xerrors.Errorf("Value in field t.Rev was too long")
77
+
}
78
+
79
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Rev))); err != nil {
80
+
return err
81
+
}
82
+
if _, err := cw.WriteString(string(t.Rev)); err != nil {
83
+
return err
84
+
}
85
+
}
86
+
87
+
// t.Sig ([]uint8) (slice)
88
+
if len("sig") > 1000000 {
89
+
return xerrors.Errorf("Value in field \"sig\" was too long")
90
+
}
91
+
92
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sig"))); err != nil {
93
+
return err
94
+
}
95
+
if _, err := cw.WriteString(string("sig")); err != nil {
96
+
return err
97
+
}
98
+
99
+
if len(t.Sig) > 2097152 {
100
+
return xerrors.Errorf("Byte array in field t.Sig was too long")
101
+
}
102
+
103
+
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Sig))); err != nil {
104
+
return err
105
+
}
106
+
107
+
if _, err := cw.Write(t.Sig); err != nil {
108
+
return err
109
+
}
110
+
111
+
// t.Data (cid.Cid) (struct)
112
+
if len("data") > 1000000 {
113
+
return xerrors.Errorf("Value in field \"data\" was too long")
114
+
}
115
+
116
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("data"))); err != nil {
117
+
return err
118
+
}
119
+
if _, err := cw.WriteString(string("data")); err != nil {
120
+
return err
121
+
}
122
+
123
+
if err := cbg.WriteCid(cw, t.Data); err != nil {
124
+
return xerrors.Errorf("failed to write cid field t.Data: %w", err)
125
+
}
126
+
127
+
// t.Prev (cid.Cid) (struct)
128
+
if len("prev") > 1000000 {
129
+
return xerrors.Errorf("Value in field \"prev\" was too long")
130
+
}
131
+
132
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("prev"))); err != nil {
133
+
return err
134
+
}
135
+
if _, err := cw.WriteString(string("prev")); err != nil {
136
+
return err
137
+
}
138
+
139
+
if t.Prev == nil {
140
+
if _, err := cw.Write(cbg.CborNull); err != nil {
141
+
return err
142
+
}
143
+
} else {
144
+
if err := cbg.WriteCid(cw, *t.Prev); err != nil {
145
+
return xerrors.Errorf("failed to write cid field t.Prev: %w", err)
146
+
}
147
+
}
148
+
149
+
// t.Version (int64) (int64)
150
+
if len("version") > 1000000 {
151
+
return xerrors.Errorf("Value in field \"version\" was too long")
152
+
}
153
+
154
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("version"))); err != nil {
155
+
return err
156
+
}
157
+
if _, err := cw.WriteString(string("version")); err != nil {
158
+
return err
159
+
}
160
+
161
+
if t.Version >= 0 {
162
+
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Version)); err != nil {
163
+
return err
164
+
}
165
+
} else {
166
+
if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Version-1)); err != nil {
167
+
return err
168
+
}
169
+
}
170
+
171
+
return nil
172
+
}
173
+
174
+
func (t *Commit) UnmarshalCBOR(r io.Reader) (err error) {
175
+
*t = Commit{}
176
+
177
+
cr := cbg.NewCborReader(r)
178
+
179
+
maj, extra, err := cr.ReadHeader()
180
+
if err != nil {
181
+
return err
182
+
}
183
+
defer func() {
184
+
if err == io.EOF {
185
+
err = io.ErrUnexpectedEOF
186
+
}
187
+
}()
188
+
189
+
if maj != cbg.MajMap {
190
+
return fmt.Errorf("cbor input should be of type map")
191
+
}
192
+
193
+
if extra > cbg.MaxLength {
194
+
return fmt.Errorf("Commit: map struct too large (%d)", extra)
195
+
}
196
+
197
+
n := extra
198
+
199
+
nameBuf := make([]byte, 7)
200
+
for i := uint64(0); i < n; i++ {
201
+
nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)
202
+
if err != nil {
203
+
return err
204
+
}
205
+
206
+
if !ok {
207
+
// Field doesn't exist on this type, so ignore it
208
+
if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil {
209
+
return err
210
+
}
211
+
continue
212
+
}
213
+
214
+
switch string(nameBuf[:nameLen]) {
215
+
// t.DID (string) (string)
216
+
case "did":
217
+
218
+
{
219
+
sval, err := cbg.ReadStringWithMax(cr, 1000000)
220
+
if err != nil {
221
+
return err
222
+
}
223
+
224
+
t.DID = string(sval)
225
+
}
226
+
// t.Rev (string) (string)
227
+
case "rev":
228
+
229
+
{
230
+
sval, err := cbg.ReadStringWithMax(cr, 1000000)
231
+
if err != nil {
232
+
return err
233
+
}
234
+
235
+
t.Rev = string(sval)
236
+
}
237
+
// t.Sig ([]uint8) (slice)
238
+
case "sig":
239
+
240
+
maj, extra, err = cr.ReadHeader()
241
+
if err != nil {
242
+
return err
243
+
}
244
+
245
+
if extra > 2097152 {
246
+
return fmt.Errorf("t.Sig: byte array too large (%d)", extra)
247
+
}
248
+
if maj != cbg.MajByteString {
249
+
return fmt.Errorf("expected byte array")
250
+
}
251
+
252
+
if extra > 0 {
253
+
t.Sig = make([]uint8, extra)
254
+
}
255
+
256
+
if _, err := io.ReadFull(cr, t.Sig); err != nil {
257
+
return err
258
+
}
259
+
260
+
// t.Data (cid.Cid) (struct)
261
+
case "data":
262
+
263
+
{
264
+
265
+
c, err := cbg.ReadCid(cr)
266
+
if err != nil {
267
+
return xerrors.Errorf("failed to read cid field t.Data: %w", err)
268
+
}
269
+
270
+
t.Data = c
271
+
272
+
}
273
+
// t.Prev (cid.Cid) (struct)
274
+
case "prev":
275
+
276
+
{
277
+
278
+
b, err := cr.ReadByte()
279
+
if err != nil {
280
+
return err
281
+
}
282
+
if b != cbg.CborNull[0] {
283
+
if err := cr.UnreadByte(); err != nil {
284
+
return err
285
+
}
286
+
287
+
c, err := cbg.ReadCid(cr)
288
+
if err != nil {
289
+
return xerrors.Errorf("failed to read cid field t.Prev: %w", err)
290
+
}
291
+
292
+
t.Prev = &c
293
+
}
294
+
295
+
}
296
+
// t.Version (int64) (int64)
297
+
case "version":
298
+
{
299
+
maj, extra, err := cr.ReadHeader()
300
+
if err != nil {
301
+
return err
302
+
}
303
+
var extraI int64
304
+
switch maj {
305
+
case cbg.MajUnsignedInt:
306
+
extraI = int64(extra)
307
+
if extraI < 0 {
308
+
return fmt.Errorf("int64 positive overflow")
309
+
}
310
+
case cbg.MajNegativeInt:
311
+
extraI = int64(extra)
312
+
if extraI < 0 {
313
+
return fmt.Errorf("int64 negative overflow")
314
+
}
315
+
extraI = -1 - extraI
316
+
default:
317
+
return fmt.Errorf("wrong type for int64 field: %d", maj)
318
+
}
319
+
320
+
t.Version = int64(extraI)
321
+
}
322
+
323
+
default:
324
+
// Field doesn't exist on this type, so ignore it
325
+
if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil {
326
+
return err
327
+
}
328
+
}
329
+
}
330
+
331
+
return nil
332
+
}
+76
atproto/repo/cmd/repo-tool/firehose.go
+76
atproto/repo/cmd/repo-tool/firehose.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"log/slog"
8
+
"net/http"
9
+
"net/url"
10
+
"os"
11
+
12
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
13
+
"github.com/bluesky-social/indigo/atproto/repo"
14
+
"github.com/bluesky-social/indigo/events"
15
+
"github.com/bluesky-social/indigo/events/schedulers/parallel"
16
+
17
+
"github.com/carlmjohnson/versioninfo"
18
+
"github.com/gorilla/websocket"
19
+
"github.com/urfave/cli/v2"
20
+
)
21
+
22
+
// write out error cases as JSON files to disk, for use in regression tests
23
+
var CAPTURE_TEST_CASES = false
24
+
25
+
func runVerifyFirehose(cctx *cli.Context) error {
26
+
ctx := context.Background()
27
+
28
+
slog.SetDefault(configLogger(cctx, os.Stdout))
29
+
30
+
relayHost := cctx.String("relay-host")
31
+
32
+
dialer := websocket.DefaultDialer
33
+
u, err := url.Parse(relayHost)
34
+
if err != nil {
35
+
return fmt.Errorf("invalid relayHost URI: %w", err)
36
+
}
37
+
u.Path = "xrpc/com.atproto.sync.subscribeRepos"
38
+
con, _, err := dialer.Dial(u.String(), http.Header{
39
+
"User-Agent": []string{fmt.Sprintf("goat/%s", versioninfo.Short())},
40
+
})
41
+
if err != nil {
42
+
return fmt.Errorf("subscribing to firehose failed (dialing): %w", err)
43
+
}
44
+
45
+
rsc := &events.RepoStreamCallbacks{
46
+
RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error {
47
+
slog.Debug("commit event", "did", evt.Repo, "seq", evt.Seq)
48
+
return handleCommitEvent(ctx, evt)
49
+
},
50
+
}
51
+
52
+
scheduler := parallel.NewScheduler(
53
+
1,
54
+
100,
55
+
relayHost,
56
+
rsc.EventHandler,
57
+
)
58
+
slog.Info("starting firehose consumer", "relayHost", relayHost)
59
+
return events.HandleRepoStream(ctx, con, scheduler, nil)
60
+
}
61
+
62
+
func handleCommitEvent(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) error {
63
+
// TODO: just log errors, not fail?
64
+
_, err := repo.VerifyCommitMessage(ctx, evt)
65
+
if err != nil && CAPTURE_TEST_CASES {
66
+
body, err := json.MarshalIndent(evt, "", " ")
67
+
if err != nil {
68
+
return err
69
+
}
70
+
p := fmt.Sprintf("firehose_commit_%d.json", evt.Seq)
71
+
if err := os.WriteFile(p, body, 0600); err != nil {
72
+
return err
73
+
}
74
+
}
75
+
return err
76
+
}
+103
atproto/repo/cmd/repo-tool/main.go
+103
atproto/repo/cmd/repo-tool/main.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"fmt"
6
+
"io"
7
+
"log/slog"
8
+
"os"
9
+
"strings"
10
+
11
+
"github.com/bluesky-social/indigo/atproto/repo"
12
+
13
+
"github.com/urfave/cli/v2"
14
+
)
15
+
16
+
func main() {
17
+
app := cli.App{
18
+
Name: "repo-tool",
19
+
Usage: "development tool for atproto MST trees, CAR files, etc",
20
+
Flags: []cli.Flag{
21
+
&cli.StringFlag{
22
+
Name: "log-level",
23
+
Usage: "log verbosity level (eg: warn, info, debug)",
24
+
EnvVars: []string{"BEEMO_LOG_LEVEL", "GO_LOG_LEVEL", "LOG_LEVEL"},
25
+
},
26
+
},
27
+
}
28
+
app.Commands = []*cli.Command{
29
+
&cli.Command{
30
+
Name: "verify-car-mst",
31
+
Usage: "load a CAR file and check the MST tree",
32
+
ArgsUsage: "<path>",
33
+
Action: runVerifyCarMst,
34
+
},
35
+
&cli.Command{
36
+
Name: "verify-firehose",
37
+
Usage: "subscribes to sync firehose and validates commit messages",
38
+
Action: runVerifyFirehose,
39
+
Flags: []cli.Flag{
40
+
&cli.StringFlag{
41
+
Name: "relay-host",
42
+
Usage: "method, hostname, and port of Relay instance (websocket)",
43
+
Value: "wss://bsky.network",
44
+
EnvVars: []string{"ATP_RELAY_HOST"},
45
+
},
46
+
},
47
+
},
48
+
}
49
+
h := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelDebug})
50
+
slog.SetDefault(slog.New(h))
51
+
app.RunAndExitOnError()
52
+
}
53
+
54
+
func configLogger(cctx *cli.Context, writer io.Writer) *slog.Logger {
55
+
var level slog.Level
56
+
switch strings.ToLower(cctx.String("log-level")) {
57
+
case "error":
58
+
level = slog.LevelError
59
+
case "warn":
60
+
level = slog.LevelWarn
61
+
case "info":
62
+
level = slog.LevelInfo
63
+
case "debug":
64
+
level = slog.LevelDebug
65
+
default:
66
+
level = slog.LevelInfo
67
+
}
68
+
logger := slog.New(slog.NewJSONHandler(writer, &slog.HandlerOptions{
69
+
Level: level,
70
+
}))
71
+
slog.SetDefault(logger)
72
+
return logger
73
+
}
74
+
75
+
func runVerifyCarMst(cctx *cli.Context) error {
76
+
ctx := context.Background()
77
+
p := cctx.Args().First()
78
+
if p == "" {
79
+
return fmt.Errorf("need to provide path to CAR file")
80
+
}
81
+
82
+
f, err := os.Open(p)
83
+
if err != nil {
84
+
return err
85
+
}
86
+
defer f.Close()
87
+
88
+
repo, err := repo.LoadFromCAR(ctx, f)
89
+
if err != nil {
90
+
return err
91
+
}
92
+
93
+
computedCID, err := repo.MST.RootCID()
94
+
if err != nil {
95
+
return err
96
+
}
97
+
98
+
if repo.Commit.Data != *computedCID {
99
+
return fmt.Errorf("failed to re-compute: %s != %s", computedCID, repo.Commit.Data)
100
+
}
101
+
fmt.Println("verified tree")
102
+
return nil
103
+
}
+37
atproto/repo/commit.go
+37
atproto/repo/commit.go
···
1
+
package repo
2
+
3
+
import (
4
+
"fmt"
5
+
6
+
"github.com/bluesky-social/indigo/atproto/syntax"
7
+
8
+
"github.com/ipfs/go-cid"
9
+
)
10
+
11
+
type Commit struct {
12
+
DID string `json:"did" cborgen:"did"`
13
+
Version int64 `json:"version" cborgen:"version"` // currently: 3
14
+
Prev *cid.Cid `json:"prev" cborgen:"prev"` // TODO: could we omitempty yet? breaks signatures I guess
15
+
Data cid.Cid `json:"data" cborgen:"data"`
16
+
Sig []byte `json:"sig" cborgen:"sig"`
17
+
Rev string `json:"rev" cborgen:"rev"`
18
+
}
19
+
20
+
// does basic checks that syntax is correct
21
+
func (c *Commit) VerifyStructure() error {
22
+
if c.Version != ATPROTO_REPO_VERSION {
23
+
return fmt.Errorf("unsupported repo version: %d", c.Version)
24
+
}
25
+
if len(c.Sig) == 0 {
26
+
return fmt.Errorf("empty commit signature")
27
+
}
28
+
_, err := syntax.ParseDID(c.DID)
29
+
if err != nil {
30
+
return fmt.Errorf("invalid commit data: %w", err)
31
+
}
32
+
_, err = syntax.ParseTID(c.Rev)
33
+
if err != nil {
34
+
return fmt.Errorf("invalid commit data: %w", err)
35
+
}
36
+
return nil
37
+
}
+433
atproto/repo/mst/cbor_gen.go
+433
atproto/repo/mst/cbor_gen.go
···
1
+
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
2
+
3
+
package mst
4
+
5
+
import (
6
+
"fmt"
7
+
"io"
8
+
"math"
9
+
"sort"
10
+
11
+
cid "github.com/ipfs/go-cid"
12
+
cbg "github.com/whyrusleeping/cbor-gen"
13
+
xerrors "golang.org/x/xerrors"
14
+
)
15
+
16
+
var _ = xerrors.Errorf
17
+
var _ = cid.Undef
18
+
var _ = math.E
19
+
var _ = sort.Sort
20
+
21
+
func (t *NodeData) MarshalCBOR(w io.Writer) error {
22
+
if t == nil {
23
+
_, err := w.Write(cbg.CborNull)
24
+
return err
25
+
}
26
+
27
+
cw := cbg.NewCborWriter(w)
28
+
29
+
if _, err := cw.Write([]byte{162}); err != nil {
30
+
return err
31
+
}
32
+
33
+
// t.Entries ([]mst.EntryData) (slice)
34
+
if len("e") > 1000000 {
35
+
return xerrors.Errorf("Value in field \"e\" was too long")
36
+
}
37
+
38
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("e"))); err != nil {
39
+
return err
40
+
}
41
+
if _, err := cw.WriteString(string("e")); err != nil {
42
+
return err
43
+
}
44
+
45
+
if len(t.Entries) > 8192 {
46
+
return xerrors.Errorf("Slice value in field t.Entries was too long")
47
+
}
48
+
49
+
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Entries))); err != nil {
50
+
return err
51
+
}
52
+
for _, v := range t.Entries {
53
+
if err := v.MarshalCBOR(cw); err != nil {
54
+
return err
55
+
}
56
+
57
+
}
58
+
59
+
// t.Left (cid.Cid) (struct)
60
+
if len("l") > 1000000 {
61
+
return xerrors.Errorf("Value in field \"l\" was too long")
62
+
}
63
+
64
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("l"))); err != nil {
65
+
return err
66
+
}
67
+
if _, err := cw.WriteString(string("l")); err != nil {
68
+
return err
69
+
}
70
+
71
+
if t.Left == nil {
72
+
if _, err := cw.Write(cbg.CborNull); err != nil {
73
+
return err
74
+
}
75
+
} else {
76
+
if err := cbg.WriteCid(cw, *t.Left); err != nil {
77
+
return xerrors.Errorf("failed to write cid field t.Left: %w", err)
78
+
}
79
+
}
80
+
81
+
return nil
82
+
}
83
+
84
+
func (t *NodeData) UnmarshalCBOR(r io.Reader) (err error) {
85
+
*t = NodeData{}
86
+
87
+
cr := cbg.NewCborReader(r)
88
+
89
+
maj, extra, err := cr.ReadHeader()
90
+
if err != nil {
91
+
return err
92
+
}
93
+
defer func() {
94
+
if err == io.EOF {
95
+
err = io.ErrUnexpectedEOF
96
+
}
97
+
}()
98
+
99
+
if maj != cbg.MajMap {
100
+
return fmt.Errorf("cbor input should be of type map")
101
+
}
102
+
103
+
if extra > cbg.MaxLength {
104
+
return fmt.Errorf("NodeData: map struct too large (%d)", extra)
105
+
}
106
+
107
+
n := extra
108
+
109
+
nameBuf := make([]byte, 1)
110
+
for i := uint64(0); i < n; i++ {
111
+
nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)
112
+
if err != nil {
113
+
return err
114
+
}
115
+
116
+
if !ok {
117
+
// Field doesn't exist on this type, so ignore it
118
+
if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil {
119
+
return err
120
+
}
121
+
continue
122
+
}
123
+
124
+
switch string(nameBuf[:nameLen]) {
125
+
// t.Entries ([]mst.EntryData) (slice)
126
+
case "e":
127
+
128
+
maj, extra, err = cr.ReadHeader()
129
+
if err != nil {
130
+
return err
131
+
}
132
+
133
+
if extra > 8192 {
134
+
return fmt.Errorf("t.Entries: array too large (%d)", extra)
135
+
}
136
+
137
+
if maj != cbg.MajArray {
138
+
return fmt.Errorf("expected cbor array")
139
+
}
140
+
141
+
if extra > 0 {
142
+
t.Entries = make([]EntryData, extra)
143
+
}
144
+
145
+
for i := 0; i < int(extra); i++ {
146
+
{
147
+
var maj byte
148
+
var extra uint64
149
+
var err error
150
+
_ = maj
151
+
_ = extra
152
+
_ = err
153
+
154
+
{
155
+
156
+
if err := t.Entries[i].UnmarshalCBOR(cr); err != nil {
157
+
return xerrors.Errorf("unmarshaling t.Entries[i]: %w", err)
158
+
}
159
+
160
+
}
161
+
162
+
}
163
+
}
164
+
// t.Left (cid.Cid) (struct)
165
+
case "l":
166
+
167
+
{
168
+
169
+
b, err := cr.ReadByte()
170
+
if err != nil {
171
+
return err
172
+
}
173
+
if b != cbg.CborNull[0] {
174
+
if err := cr.UnreadByte(); err != nil {
175
+
return err
176
+
}
177
+
178
+
c, err := cbg.ReadCid(cr)
179
+
if err != nil {
180
+
return xerrors.Errorf("failed to read cid field t.Left: %w", err)
181
+
}
182
+
183
+
t.Left = &c
184
+
}
185
+
186
+
}
187
+
188
+
default:
189
+
// Field doesn't exist on this type, so ignore it
190
+
if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil {
191
+
return err
192
+
}
193
+
}
194
+
}
195
+
196
+
return nil
197
+
}
198
+
func (t *EntryData) MarshalCBOR(w io.Writer) error {
199
+
if t == nil {
200
+
_, err := w.Write(cbg.CborNull)
201
+
return err
202
+
}
203
+
204
+
cw := cbg.NewCborWriter(w)
205
+
206
+
if _, err := cw.Write([]byte{164}); err != nil {
207
+
return err
208
+
}
209
+
210
+
// t.KeySuffix ([]uint8) (slice)
211
+
if len("k") > 1000000 {
212
+
return xerrors.Errorf("Value in field \"k\" was too long")
213
+
}
214
+
215
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("k"))); err != nil {
216
+
return err
217
+
}
218
+
if _, err := cw.WriteString(string("k")); err != nil {
219
+
return err
220
+
}
221
+
222
+
if len(t.KeySuffix) > 2097152 {
223
+
return xerrors.Errorf("Byte array in field t.KeySuffix was too long")
224
+
}
225
+
226
+
if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.KeySuffix))); err != nil {
227
+
return err
228
+
}
229
+
230
+
if _, err := cw.Write(t.KeySuffix); err != nil {
231
+
return err
232
+
}
233
+
234
+
// t.PrefixLen (int64) (int64)
235
+
if len("p") > 1000000 {
236
+
return xerrors.Errorf("Value in field \"p\" was too long")
237
+
}
238
+
239
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("p"))); err != nil {
240
+
return err
241
+
}
242
+
if _, err := cw.WriteString(string("p")); err != nil {
243
+
return err
244
+
}
245
+
246
+
if t.PrefixLen >= 0 {
247
+
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.PrefixLen)); err != nil {
248
+
return err
249
+
}
250
+
} else {
251
+
if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.PrefixLen-1)); err != nil {
252
+
return err
253
+
}
254
+
}
255
+
256
+
// t.Right (cid.Cid) (struct)
257
+
if len("t") > 1000000 {
258
+
return xerrors.Errorf("Value in field \"t\" was too long")
259
+
}
260
+
261
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("t"))); err != nil {
262
+
return err
263
+
}
264
+
if _, err := cw.WriteString(string("t")); err != nil {
265
+
return err
266
+
}
267
+
268
+
if t.Right == nil {
269
+
if _, err := cw.Write(cbg.CborNull); err != nil {
270
+
return err
271
+
}
272
+
} else {
273
+
if err := cbg.WriteCid(cw, *t.Right); err != nil {
274
+
return xerrors.Errorf("failed to write cid field t.Right: %w", err)
275
+
}
276
+
}
277
+
278
+
// t.Value (cid.Cid) (struct)
279
+
if len("v") > 1000000 {
280
+
return xerrors.Errorf("Value in field \"v\" was too long")
281
+
}
282
+
283
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("v"))); err != nil {
284
+
return err
285
+
}
286
+
if _, err := cw.WriteString(string("v")); err != nil {
287
+
return err
288
+
}
289
+
290
+
if err := cbg.WriteCid(cw, t.Value); err != nil {
291
+
return xerrors.Errorf("failed to write cid field t.Value: %w", err)
292
+
}
293
+
294
+
return nil
295
+
}
296
+
297
+
func (t *EntryData) UnmarshalCBOR(r io.Reader) (err error) {
298
+
*t = EntryData{}
299
+
300
+
cr := cbg.NewCborReader(r)
301
+
302
+
maj, extra, err := cr.ReadHeader()
303
+
if err != nil {
304
+
return err
305
+
}
306
+
defer func() {
307
+
if err == io.EOF {
308
+
err = io.ErrUnexpectedEOF
309
+
}
310
+
}()
311
+
312
+
if maj != cbg.MajMap {
313
+
return fmt.Errorf("cbor input should be of type map")
314
+
}
315
+
316
+
if extra > cbg.MaxLength {
317
+
return fmt.Errorf("EntryData: map struct too large (%d)", extra)
318
+
}
319
+
320
+
n := extra
321
+
322
+
nameBuf := make([]byte, 1)
323
+
for i := uint64(0); i < n; i++ {
324
+
nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000)
325
+
if err != nil {
326
+
return err
327
+
}
328
+
329
+
if !ok {
330
+
// Field doesn't exist on this type, so ignore it
331
+
if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil {
332
+
return err
333
+
}
334
+
continue
335
+
}
336
+
337
+
switch string(nameBuf[:nameLen]) {
338
+
// t.KeySuffix ([]uint8) (slice)
339
+
case "k":
340
+
341
+
maj, extra, err = cr.ReadHeader()
342
+
if err != nil {
343
+
return err
344
+
}
345
+
346
+
if extra > 2097152 {
347
+
return fmt.Errorf("t.KeySuffix: byte array too large (%d)", extra)
348
+
}
349
+
if maj != cbg.MajByteString {
350
+
return fmt.Errorf("expected byte array")
351
+
}
352
+
353
+
if extra > 0 {
354
+
t.KeySuffix = make([]uint8, extra)
355
+
}
356
+
357
+
if _, err := io.ReadFull(cr, t.KeySuffix); err != nil {
358
+
return err
359
+
}
360
+
361
+
// t.PrefixLen (int64) (int64)
362
+
case "p":
363
+
{
364
+
maj, extra, err := cr.ReadHeader()
365
+
if err != nil {
366
+
return err
367
+
}
368
+
var extraI int64
369
+
switch maj {
370
+
case cbg.MajUnsignedInt:
371
+
extraI = int64(extra)
372
+
if extraI < 0 {
373
+
return fmt.Errorf("int64 positive overflow")
374
+
}
375
+
case cbg.MajNegativeInt:
376
+
extraI = int64(extra)
377
+
if extraI < 0 {
378
+
return fmt.Errorf("int64 negative overflow")
379
+
}
380
+
extraI = -1 - extraI
381
+
default:
382
+
return fmt.Errorf("wrong type for int64 field: %d", maj)
383
+
}
384
+
385
+
t.PrefixLen = int64(extraI)
386
+
}
387
+
// t.Right (cid.Cid) (struct)
388
+
case "t":
389
+
390
+
{
391
+
392
+
b, err := cr.ReadByte()
393
+
if err != nil {
394
+
return err
395
+
}
396
+
if b != cbg.CborNull[0] {
397
+
if err := cr.UnreadByte(); err != nil {
398
+
return err
399
+
}
400
+
401
+
c, err := cbg.ReadCid(cr)
402
+
if err != nil {
403
+
return xerrors.Errorf("failed to read cid field t.Right: %w", err)
404
+
}
405
+
406
+
t.Right = &c
407
+
}
408
+
409
+
}
410
+
// t.Value (cid.Cid) (struct)
411
+
case "v":
412
+
413
+
{
414
+
415
+
c, err := cbg.ReadCid(cr)
416
+
if err != nil {
417
+
return xerrors.Errorf("failed to read cid field t.Value: %w", err)
418
+
}
419
+
420
+
t.Value = c
421
+
422
+
}
423
+
424
+
default:
425
+
// Field doesn't exist on this type, so ignore it
426
+
if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil {
427
+
return err
428
+
}
429
+
}
430
+
}
431
+
432
+
return nil
433
+
}
+117
atproto/repo/mst/debug.go
+117
atproto/repo/mst/debug.go
···
1
+
package mst
2
+
3
+
import (
4
+
"fmt"
5
+
"sort"
6
+
7
+
"github.com/ipfs/go-cid"
8
+
)
9
+
10
+
func debugPrintMap(m map[string]cid.Cid) {
11
+
keys := make([]string, len(m))
12
+
i := 0
13
+
for k := range m {
14
+
keys[i] = k
15
+
i++
16
+
}
17
+
sort.Strings(keys)
18
+
for _, k := range keys {
19
+
fmt.Printf("%s\t%s\n", k, m[k])
20
+
}
21
+
}
22
+
23
+
// This function is not very well implemented or correct. Should probably switch to Devin's `goat repo mst` code.
24
+
func DebugPrintTree(n *Node, depth int) {
25
+
if n == nil {
26
+
fmt.Printf("EMPTY TREE")
27
+
return
28
+
}
29
+
if depth == 0 {
30
+
fmt.Printf("tree root (height=%d)\n", n.Height)
31
+
}
32
+
for i, e := range n.Entries {
33
+
if depth > 0 && i == 0 {
34
+
if len(n.Entries) > 1 {
35
+
fmt.Printf("┬")
36
+
} else {
37
+
fmt.Printf("─")
38
+
}
39
+
} else {
40
+
for range depth {
41
+
fmt.Printf("│")
42
+
}
43
+
if i+1 == len(n.Entries) {
44
+
fmt.Printf("└")
45
+
} else {
46
+
fmt.Printf("├")
47
+
}
48
+
}
49
+
if e.IsValue() {
50
+
fmt.Printf(" (%d) %s -> %s\n", HeightForKey(e.Key), e.Key, e.Value)
51
+
} else if e.IsChild() {
52
+
if e.Child != nil {
53
+
DebugPrintTree(e.Child, depth+1)
54
+
} else {
55
+
fmt.Printf("─ (%d; partial) %s\n", n.Height-1, e.ChildCID)
56
+
}
57
+
} else {
58
+
fmt.Printf(" BAD NODE\n")
59
+
}
60
+
}
61
+
}
62
+
63
+
func debugCountEntries(n *Node) int {
64
+
if n == nil {
65
+
return 0
66
+
}
67
+
count := 0
68
+
for _, e := range n.Entries {
69
+
if e.IsValue() {
70
+
count++
71
+
}
72
+
if e.IsChild() && e.Child != nil {
73
+
count += debugCountEntries(e.Child)
74
+
}
75
+
}
76
+
return count
77
+
}
78
+
79
+
func debugPrintNodePointers(n *Node) {
80
+
if n == nil {
81
+
return
82
+
}
83
+
fmt.Printf("%p %p\n", n, n.Entries)
84
+
for _, e := range n.Entries {
85
+
if e.IsChild() && e.Child != nil {
86
+
debugPrintNodePointers(e.Child)
87
+
}
88
+
}
89
+
}
90
+
91
+
func debugPrintChildPointers(n *Node) {
92
+
if n == nil {
93
+
return
94
+
}
95
+
for _, e := range n.Entries {
96
+
if e.IsChild() && e.Child != nil {
97
+
fmt.Printf("CHILD PTR: %p entry: %p\n", e.Child, &e)
98
+
debugPrintChildPointers(e.Child)
99
+
}
100
+
}
101
+
}
102
+
103
+
func debugSiblingChild(n *Node) error {
104
+
lastChild := false
105
+
for _, e := range n.Entries {
106
+
if e.IsChild() {
107
+
if lastChild {
108
+
return fmt.Errorf("neighboring children in entries list")
109
+
}
110
+
lastChild = true
111
+
}
112
+
if e.IsValue() {
113
+
lastChild = false
114
+
}
115
+
}
116
+
return nil
117
+
}
+34
atproto/repo/mst/doc.go
+34
atproto/repo/mst/doc.go
···
1
+
/*
2
+
Implementation of the Merkle Search Tree (MST) data structure for atproto.
3
+
4
+
## Terminology
5
+
6
+
node: any node in the tree. nodes can contain multiple entries. they should never be entirely "empty", unless the entire tree is a single empty node, but they might only contain "child" pointers
7
+
8
+
entry: nodes contain multiple entries. these can include both key/CID pairs, or pointers to child nodes. entries are always lexically sorted, with "child" entries pointing to nodes containing (recursively) keys in the appropriate lexical range. there should never be multiple "child" entries adjacent in a single node (they should be merged instead)
9
+
10
+
tree: an overall tree of nodes
11
+
12
+
## Tricky Bits
13
+
14
+
When inserting:
15
+
16
+
- the inserted key might be on a "higher" layer than the current top of the tree, in which case new parent tree nodes need to be created
17
+
- "parent" or "child" insertions might be multiple layers away from the starting node, with intermediate nodes created
18
+
- inserting a "value" entry in a node might require "splitting" a child node, if the key on the current layer would have fallen within the lexical range of the child
19
+
20
+
When removing:
21
+
22
+
- deleting an entry from a node might result in a "merge" of two child nodes which are no longer "split"
23
+
- removing a "value" entry from the top of the tree might make it a simple pointer down to a child. in this case the top of the tree should be "trimmed" (this might involve multiple layers of trimming)
24
+
25
+
When inverting operations:
26
+
27
+
- need additional "proof" blocks to invert deletions. basically need the proof blocks for any keys (at any layer) directly adjacent to the deleted block
28
+
- if an entry is removed from the top of a partial tree and results in "trimming", and the child node is not available, the overall tree root CID might still be known
29
+
30
+
## Hacking
31
+
32
+
Be careful with go slices. Need to avoid creating multiple references (slices) of the same underlying array, which can lead to "mutation at a distance" in ways that are hard to debug.
33
+
*/
34
+
package mst
+234
atproto/repo/mst/encoding.go
+234
atproto/repo/mst/encoding.go
···
1
+
package mst
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"io"
8
+
9
+
bf "github.com/ipfs/go-block-format"
10
+
"github.com/ipfs/go-cid"
11
+
blockstore "github.com/ipfs/go-ipfs-blockstore"
12
+
ipld "github.com/ipfs/go-ipld-format"
13
+
"github.com/multiformats/go-multihash"
14
+
)
15
+
16
+
// CBOR serialization struct for a MST tree node. MST tree node as gets serialized to CBOR. Note that the CBOR fields are all single-character.
17
+
type NodeData struct {
18
+
Left *cid.Cid `cborgen:"l"` // [nullable] pointer to lower-level subtree to the "left" of this path/key
19
+
Entries []EntryData `cborgen:"e"` // ordered list of entries at this node
20
+
}
21
+
22
+
// CBOR serialization struct for a single entry within a `NodeData` entry list.
23
+
type EntryData struct {
24
+
PrefixLen int64 `cborgen:"p"` // count of characters shared with previous path/key in tree
25
+
KeySuffix []byte `cborgen:"k"` // remaining part of path/key (appended to "previous key")
26
+
Value cid.Cid `cborgen:"v"` // CID pointer at this path/key
27
+
Right *cid.Cid `cborgen:"t"` // [nullable] pointer to lower-level subtree to the "right" of this path/key entry
28
+
}
29
+
30
+
// Encodes a single `NodeData` struct as CBOR bytes. Does not recursively encode or update children.
31
+
func (d *NodeData) Bytes() ([]byte, *cid.Cid, error) {
32
+
buf := new(bytes.Buffer)
33
+
if err := d.MarshalCBOR(buf); err != nil {
34
+
return nil, nil, err
35
+
}
36
+
b := buf.Bytes()
37
+
builder := cid.NewPrefixV1(cid.DagCBOR, multihash.SHA2_256)
38
+
c, err := builder.Sum(b)
39
+
if err != nil {
40
+
return nil, nil, err
41
+
}
42
+
return b, &c, nil
43
+
}
44
+
45
+
// Parses CBOR bytes in to `NodeData` struct
46
+
func NodeDataFromCBOR(r io.Reader) (*NodeData, error) {
47
+
var nd NodeData
48
+
if err := nd.UnmarshalCBOR(r); err != nil {
49
+
return nil, err
50
+
}
51
+
// TODO: verify CID type, and "non-empty" here?
52
+
return &nd, nil
53
+
}
54
+
55
+
// Transforms `Node` stuct to `NodeData`, which is the format used for encoding to CBOR.
56
+
//
57
+
// Will panic if any entries are missing a CID (must compute those first)
58
+
func (n *Node) NodeData() NodeData {
59
+
d := NodeData{
60
+
Left: nil,
61
+
Entries: []EntryData{}, // TODO perf: pre-allocate an array
62
+
}
63
+
64
+
prevKey := []byte{}
65
+
for i, e := range n.Entries {
66
+
if i == 0 && e.IsChild() {
67
+
d.Left = e.ChildCID
68
+
continue
69
+
}
70
+
if e.IsChild() {
71
+
if len(d.Entries) == 0 {
72
+
panic("malformed tree node") // TODO: return error?
73
+
}
74
+
d.Entries[len(d.Entries)-1].Right = e.ChildCID
75
+
}
76
+
if e.IsValue() {
77
+
idx := int64(CountPrefixLen(prevKey, e.Key))
78
+
d.Entries = append(d.Entries, EntryData{
79
+
PrefixLen: idx,
80
+
KeySuffix: e.Key[idx:],
81
+
Value: *e.Value,
82
+
Right: nil,
83
+
})
84
+
prevKey = e.Key
85
+
}
86
+
}
87
+
return d
88
+
}
89
+
90
+
// Tansforms an encoded `NodeData` to `Node` data structure format.
91
+
//
92
+
// c: optional CID argument for the CID of the CBOR representation of the NodeData
93
+
func (d *NodeData) Node(c *cid.Cid) Node {
94
+
height := -1
95
+
n := Node{
96
+
CID: c,
97
+
Dirty: c == nil,
98
+
Entries: []NodeEntry{}, // TODO: pre-allocate
99
+
}
100
+
101
+
if d.Left != nil {
102
+
n.Entries = append(n.Entries, NodeEntry{ChildCID: d.Left})
103
+
}
104
+
105
+
var prevKey []byte
106
+
for _, e := range d.Entries {
107
+
// TODO perf: pre-allocate
108
+
key := []byte{}
109
+
key = append(key, prevKey[:e.PrefixLen]...)
110
+
key = append(key, e.KeySuffix...)
111
+
n.Entries = append(n.Entries, NodeEntry{
112
+
Key: key,
113
+
Value: &e.Value,
114
+
})
115
+
prevKey = key
116
+
if height < 0 {
117
+
height = HeightForKey(key)
118
+
}
119
+
120
+
if e.Right != nil {
121
+
n.Entries = append(n.Entries, NodeEntry{
122
+
ChildCID: e.Right,
123
+
})
124
+
}
125
+
}
126
+
127
+
// TODO: height doesn't get set properly if this is an intermediate node; we rely on `EnsureHeights` getting called to fix that
128
+
n.Height = height
129
+
return n
130
+
}
131
+
132
+
// TODO: this feels like a hack, and easy to forget
133
+
func (n *Node) ensureHeights() {
134
+
if n.Height <= 0 {
135
+
return
136
+
}
137
+
for _, e := range n.Entries {
138
+
if e.Child != nil {
139
+
if n.Height > 0 && e.Child.Height < 0 {
140
+
e.Child.Height = n.Height - 1
141
+
}
142
+
e.Child.ensureHeights()
143
+
}
144
+
}
145
+
}
146
+
147
+
// Recursively encodes sub-tree, optionally writing to blockstore. Returns root CID.
148
+
//
149
+
// This method will not error if tree is partial.
150
+
//
151
+
// bs: is an optional blockstore; if it is nil, blocks will not be written.
152
+
// onlyDirty: is an optional blockstore; if it is nil, blocks will not be written.
153
+
func (n *Node) writeBlocks(ctx context.Context, bs blockstore.Blockstore, onlyDirty bool) (*cid.Cid, error) {
154
+
if n == nil || n.Stub {
155
+
return nil, fmt.Errorf("%w: nil tree node", ErrInvalidTree)
156
+
}
157
+
if onlyDirty && !n.Dirty && n.CID != nil {
158
+
return n.CID, nil
159
+
}
160
+
161
+
// walk all children first
162
+
for i, e := range n.Entries {
163
+
if e.IsValue() && e.Dirty {
164
+
// TODO: should we actually clear this here?
165
+
e.Dirty = false
166
+
}
167
+
if !e.IsChild() {
168
+
continue
169
+
}
170
+
if e.Child != nil && (e.Dirty || e.Child.Dirty || !onlyDirty) {
171
+
cc, err := e.Child.writeBlocks(ctx, bs, onlyDirty)
172
+
if err != nil {
173
+
return nil, err
174
+
}
175
+
n.Entries[i].ChildCID = cc
176
+
n.Entries[i].Dirty = false
177
+
}
178
+
}
179
+
180
+
// compute this block
181
+
nd := n.NodeData()
182
+
b, c, err := nd.Bytes()
183
+
if err != nil {
184
+
return nil, err
185
+
}
186
+
187
+
n.CID = c
188
+
n.Dirty = false
189
+
190
+
if bs != nil {
191
+
blk, err := bf.NewBlockWithCid(b, *c)
192
+
if err != nil {
193
+
return nil, err
194
+
}
195
+
if err := bs.Put(ctx, blk); err != nil {
196
+
return nil, err
197
+
}
198
+
}
199
+
return c, nil
200
+
}
201
+
202
+
func loadNodeFromStore(ctx context.Context, bs blockstore.Blockstore, ref cid.Cid) (*Node, error) {
203
+
block, err := bs.Get(ctx, ref)
204
+
if err != nil {
205
+
return nil, err
206
+
}
207
+
208
+
nd, err := NodeDataFromCBOR(bytes.NewReader(block.RawData()))
209
+
if err != nil {
210
+
return nil, err
211
+
}
212
+
213
+
n := nd.Node(&ref)
214
+
215
+
for i, e := range n.Entries {
216
+
if e.IsChild() {
217
+
child, err := loadNodeFromStore(ctx, bs, *e.ChildCID)
218
+
if err != nil && ipld.IsNotFound(err) {
219
+
// allow "partial" trees
220
+
continue
221
+
}
222
+
if err != nil {
223
+
return nil, err
224
+
}
225
+
n.Entries[i].Child = child
226
+
// NOTE: this is kind of a hack
227
+
if n.Height == -1 && child.Height >= 0 {
228
+
n.Height = child.Height + 1
229
+
}
230
+
}
231
+
}
232
+
233
+
return &n, nil
234
+
}
+308
atproto/repo/mst/mst_interop_test.go
+308
atproto/repo/mst/mst_interop_test.go
···
1
+
// This file contains tests which are the same across language implementations.
2
+
// AKA, if you update this file, you should probably update the corresponding
3
+
// file in atproto repo (typescript)
4
+
package mst
5
+
6
+
import (
7
+
"context"
8
+
"testing"
9
+
10
+
"github.com/stretchr/testify/assert"
11
+
12
+
"github.com/ipfs/go-cid"
13
+
)
14
+
15
+
func mapToCidMapDecode(t *testing.T, a map[string]string) map[string]cid.Cid {
16
+
out := make(map[string]cid.Cid)
17
+
for k, v := range a {
18
+
c, err := cid.Decode(v)
19
+
if err != nil {
20
+
t.Fatal(err)
21
+
}
22
+
out[k] = c
23
+
}
24
+
return out
25
+
}
26
+
27
+
func mapToTreeRootCidString(t *testing.T, m map[string]string) string {
28
+
29
+
tree, err := LoadTreeFromMap(mapToCidMapDecode(t, m))
30
+
if err != nil {
31
+
t.Fatal(err)
32
+
}
33
+
34
+
c, err := tree.RootCID()
35
+
if err != nil {
36
+
t.Fatal(err)
37
+
}
38
+
39
+
return c.String()
40
+
}
41
+
42
+
// TODO: TestAllowedKeys
43
+
44
+
func TestManualNode(t *testing.T) {
45
+
assert := assert.New(t)
46
+
47
+
cid1, err := cid.Decode("bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454")
48
+
if err != nil {
49
+
t.Fatal(err)
50
+
}
51
+
52
+
simple_nd := NodeData{
53
+
Left: nil,
54
+
Entries: []EntryData{
55
+
{
56
+
PrefixLen: 0,
57
+
KeySuffix: []byte("com.example.record/3jqfcqzm3fo2j"),
58
+
Value: cid1,
59
+
Right: nil,
60
+
},
61
+
},
62
+
}
63
+
n := simple_nd.Node(nil)
64
+
assert.Equal(simple_nd, n.NodeData())
65
+
66
+
mcid, err := n.writeBlocks(context.Background(), nil, true)
67
+
if err != nil {
68
+
t.Fatal(err)
69
+
}
70
+
assert.NoError(err)
71
+
assert.Equal("bafyreibj4lsc3aqnrvphp5xmrnfoorvru4wynt6lwidqbm2623a6tatzdu", mcid.String())
72
+
}
73
+
74
+
func TestInteropKnownMaps(t *testing.T) {
75
+
assert := assert.New(t)
76
+
77
+
cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454"
78
+
79
+
// empty map
80
+
emptyMap := map[string]string{}
81
+
assert.Equal("bafyreie5737gdxlw5i64vzichcalba3z2v5n6icifvx5xytvske7mr3hpm", mapToTreeRootCidString(t, emptyMap))
82
+
83
+
// no depth, single entry
84
+
trivialMap := map[string]string{
85
+
"com.example.record/3jqfcqzm3fo2j": cid1str,
86
+
}
87
+
assert.Equal("bafyreibj4lsc3aqnrvphp5xmrnfoorvru4wynt6lwidqbm2623a6tatzdu", mapToTreeRootCidString(t, trivialMap))
88
+
89
+
// single layer=2 entry
90
+
singlelayer2Map := map[string]string{
91
+
"com.example.record/3jqfcqzm3fx2j": cid1str,
92
+
}
93
+
assert.Equal("bafyreih7wfei65pxzhauoibu3ls7jgmkju4bspy4t2ha2qdjnzqvoy33ai", mapToTreeRootCidString(t, singlelayer2Map))
94
+
95
+
// pretty simple, but with some depth
96
+
simpleMap := map[string]string{
97
+
"com.example.record/3jqfcqzm3fp2j": cid1str,
98
+
"com.example.record/3jqfcqzm3fr2j": cid1str,
99
+
"com.example.record/3jqfcqzm3fs2j": cid1str,
100
+
"com.example.record/3jqfcqzm3ft2j": cid1str,
101
+
"com.example.record/3jqfcqzm4fc2j": cid1str,
102
+
}
103
+
assert.Equal("bafyreicmahysq4n6wfuxo522m6dpiy7z7qzym3dzs756t5n7nfdgccwq7m", mapToTreeRootCidString(t, simpleMap))
104
+
}
105
+
106
+
func TestInteropKnownMapsTricky(t *testing.T) {
107
+
assert := assert.New(t)
108
+
109
+
t.Skip("TODO: these are currently disallowed in typescript implementation")
110
+
111
+
cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454"
112
+
113
+
// include several known edge cases
114
+
trickyMap := map[string]string{
115
+
"": cid1str,
116
+
"jalapeño": cid1str,
117
+
"coöperative": cid1str,
118
+
"coüperative": cid1str,
119
+
"abc\x00": cid1str,
120
+
}
121
+
assert.Equal("bafyreiecb33zh7r2sc3k2wthm6exwzfktof63kmajeildktqc25xj6qzx4", mapToTreeRootCidString(t, trickyMap))
122
+
}
123
+
124
+
// "trims top of tree on delete"
125
+
func TestInteropEdgeCasesTrimTop(t *testing.T) {
126
+
assert := assert.New(t)
127
+
128
+
cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454"
129
+
l1root := "bafyreifnqrwbk6ffmyaz5qtujqrzf5qmxf7cbxvgzktl4e3gabuxbtatv4"
130
+
l0root := "bafyreie4kjuxbwkhzg2i5dljaswcroeih4dgiqq6pazcmunwt2byd725vi"
131
+
132
+
trimMap := map[string]string{
133
+
"com.example.record/3jqfcqzm3fn2j": cid1str, // level 0
134
+
"com.example.record/3jqfcqzm3fo2j": cid1str, // level 0
135
+
"com.example.record/3jqfcqzm3fp2j": cid1str, // level 0
136
+
"com.example.record/3jqfcqzm3fs2j": cid1str, // level 0
137
+
"com.example.record/3jqfcqzm3ft2j": cid1str, // level 0
138
+
"com.example.record/3jqfcqzm3fu2j": cid1str, // level 1
139
+
}
140
+
trimTree, err := LoadTreeFromMap(mapToCidMapDecode(t, trimMap))
141
+
if err != nil {
142
+
t.Fatal(err)
143
+
}
144
+
trimBefore, err := trimTree.RootCID()
145
+
if err != nil {
146
+
t.Fatal(err)
147
+
}
148
+
assert.Equal(1, trimTree.Root.Height)
149
+
assert.Equal(l1root, trimBefore.String())
150
+
151
+
_, err = trimTree.Remove([]byte("com.example.record/3jqfcqzm3fs2j")) // level 1
152
+
if err != nil {
153
+
t.Fatal(err)
154
+
}
155
+
trimAfter, err := trimTree.RootCID()
156
+
if err != nil {
157
+
t.Fatal(err)
158
+
}
159
+
//fmt.Printf("%#v\n", trimTree)
160
+
assert.Equal(0, trimTree.Root.Height)
161
+
assert.Equal(l0root, trimAfter.String())
162
+
}
163
+
164
+
func TestInteropEdgeCasesInsertion(t *testing.T) {
165
+
assert := assert.New(t)
166
+
167
+
cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454"
168
+
cid1, err := cid.Decode(cid1str)
169
+
if err != nil {
170
+
t.Fatal(err)
171
+
}
172
+
173
+
// "handles insertion that splits two layers down"
174
+
l1root := "bafyreiettyludka6fpgp33stwxfuwhkzlur6chs4d2v4nkmq2j3ogpdjem"
175
+
l2root := "bafyreid2x5eqs4w4qxvc5jiwda4cien3gw2q6cshofxwnvv7iucrmfohpm"
176
+
insertionMap := map[string]string{
177
+
"com.example.record/3jqfcqzm3fo2j": cid1str, // A; level 0
178
+
"com.example.record/3jqfcqzm3fp2j": cid1str, // B; level 0
179
+
"com.example.record/3jqfcqzm3fr2j": cid1str, // C; level 0
180
+
"com.example.record/3jqfcqzm3fs2j": cid1str, // D; level 1
181
+
"com.example.record/3jqfcqzm3ft2j": cid1str, // E; level 0
182
+
"com.example.record/3jqfcqzm3fz2j": cid1str, // G; level 0
183
+
"com.example.record/3jqfcqzm4fc2j": cid1str, // H; level 0
184
+
"com.example.record/3jqfcqzm4fd2j": cid1str, // I; level 1
185
+
"com.example.record/3jqfcqzm4ff2j": cid1str, // J; level 0
186
+
"com.example.record/3jqfcqzm4fg2j": cid1str, // K; level 0
187
+
"com.example.record/3jqfcqzm4fh2j": cid1str, // L; level 0
188
+
}
189
+
insertionTree, err := LoadTreeFromMap(mapToCidMapDecode(t, insertionMap))
190
+
if err != nil {
191
+
t.Fatal(err)
192
+
}
193
+
insertionBefore, err := insertionTree.RootCID()
194
+
if err != nil {
195
+
t.Fatal(err)
196
+
}
197
+
assert.Equal(1, insertionTree.Root.Height)
198
+
assert.Equal(l1root, insertionBefore.String())
199
+
200
+
// insert F, which will push E out of the node with G+H to a new node under D
201
+
_, err = insertionTree.Insert([]byte("com.example.record/3jqfcqzm3fx2j"), cid1) // F; level 2
202
+
if err != nil {
203
+
t.Fatal(err)
204
+
}
205
+
insertionAfter, err := insertionTree.RootCID()
206
+
if err != nil {
207
+
t.Fatal(err)
208
+
}
209
+
assert.Equal(2, insertionTree.Root.Height)
210
+
assert.Equal(l2root, insertionAfter.String())
211
+
212
+
// remove F, which should push E back over with G+H
213
+
_, err = insertionTree.Remove([]byte("com.example.record/3jqfcqzm3fx2j")) // F; level 2
214
+
if err != nil {
215
+
t.Fatal(err)
216
+
}
217
+
insertionFinal, err := insertionTree.RootCID()
218
+
if err != nil {
219
+
t.Fatal(err)
220
+
}
221
+
assert.Equal(1, insertionTree.Root.Height)
222
+
assert.Equal(l1root, insertionFinal.String())
223
+
}
224
+
225
+
// "handles new layers that are two higher than existing"
226
+
func TestInteropEdgeCasesHigher(t *testing.T) {
227
+
assert := assert.New(t)
228
+
229
+
cid1str := "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454"
230
+
cid1, err := cid.Decode(cid1str)
231
+
if err != nil {
232
+
t.Fatal(err)
233
+
}
234
+
235
+
l0root := "bafyreidfcktqnfmykz2ps3dbul35pepleq7kvv526g47xahuz3rqtptmky"
236
+
l2root := "bafyreiavxaxdz7o7rbvr3zg2liox2yww46t7g6hkehx4i4h3lwudly7dhy"
237
+
l2root2 := "bafyreig4jv3vuajbsybhyvb7gggvpwh2zszwfyttjrj6qwvcsp24h6popu"
238
+
higherMap := map[string]string{
239
+
"com.example.record/3jqfcqzm3ft2j": cid1str, // A; level 0
240
+
"com.example.record/3jqfcqzm3fz2j": cid1str, // C; level 0
241
+
}
242
+
higherTree, err := LoadTreeFromMap(mapToCidMapDecode(t, higherMap))
243
+
if err != nil {
244
+
t.Fatal(err)
245
+
}
246
+
higherBefore, err := higherTree.RootCID()
247
+
if err != nil {
248
+
t.Fatal(err)
249
+
}
250
+
assert.Equal(0, higherTree.Root.Height)
251
+
assert.Equal(l0root, higherBefore.String())
252
+
253
+
// insert B, which is two levels above
254
+
_, err = higherTree.Insert([]byte("com.example.record/3jqfcqzm3fx2j"), cid1) // B; level 2
255
+
if err != nil {
256
+
t.Fatal(err)
257
+
}
258
+
higherAfter, err := higherTree.RootCID()
259
+
if err != nil {
260
+
t.Fatal(err)
261
+
}
262
+
assert.Equal(l2root, higherAfter.String())
263
+
//debugPrintTree(higherTree, 0)
264
+
265
+
// remove B
266
+
_, err = higherTree.Remove([]byte("com.example.record/3jqfcqzm3fx2j")) // B; level 2
267
+
if err != nil {
268
+
t.Fatal(err)
269
+
}
270
+
higherAgain, err := higherTree.RootCID()
271
+
if err != nil {
272
+
t.Fatal(err)
273
+
}
274
+
assert.Equal(0, higherTree.Root.Height)
275
+
assert.Equal(l0root, higherAgain.String())
276
+
277
+
// insert B (level=2) and D (level=1)
278
+
_, err = higherTree.Insert([]byte("com.example.record/3jqfcqzm3fx2j"), cid1) // B; level 2
279
+
if err != nil {
280
+
t.Fatal(err)
281
+
}
282
+
_, err = higherTree.Insert([]byte("com.example.record/3jqfcqzm4fd2j"), cid1) // D; level 1
283
+
if err != nil {
284
+
t.Fatal(err)
285
+
}
286
+
higherYetAgain, err := higherTree.RootCID()
287
+
if err != nil {
288
+
t.Fatal(err)
289
+
}
290
+
assert.Equal(2, higherTree.Root.Height)
291
+
assert.Equal(l2root2, higherYetAgain.String())
292
+
assert.NoError(higherTree.Verify())
293
+
//debugPrintTree(higherTree, 0)
294
+
295
+
// remove D
296
+
_, err = higherTree.Remove([]byte("com.example.record/3jqfcqzm4fd2j")) // D; level 1
297
+
if err != nil {
298
+
t.Fatal(err)
299
+
}
300
+
higherFinal, err := higherTree.RootCID()
301
+
if err != nil {
302
+
t.Fatal(err)
303
+
}
304
+
assert.Equal(2, higherTree.Root.Height)
305
+
assert.Equal(l2root, higherFinal.String())
306
+
assert.NoError(higherTree.Verify())
307
+
//debugPrintTree(higherTree, 0)
308
+
}
+276
atproto/repo/mst/mst_test.go
+276
atproto/repo/mst/mst_test.go
···
1
+
package mst
2
+
3
+
import (
4
+
"bytes"
5
+
"encoding/hex"
6
+
"math/rand"
7
+
"testing"
8
+
9
+
"github.com/ipfs/go-cid"
10
+
"github.com/multiformats/go-multihash"
11
+
"github.com/stretchr/testify/assert"
12
+
)
13
+
14
+
func TestBasicMST(t *testing.T) {
15
+
assert := assert.New(t)
16
+
17
+
c2, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222")
18
+
c3, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333")
19
+
assert.NotEmpty(c2)
20
+
assert.NotEmpty(c3)
21
+
tree := NewEmptyTree()
22
+
23
+
prev, err := tree.Insert([]byte("abc"), c2)
24
+
assert.NoError(err)
25
+
assert.Empty(prev)
26
+
27
+
assert.Equal(1, len(tree.Root.Entries))
28
+
29
+
val, err := tree.Get([]byte("abc"))
30
+
assert.NoError(err)
31
+
assert.Equal(c2, *val)
32
+
33
+
val, err = tree.Get([]byte("xyz"))
34
+
assert.NoError(err)
35
+
assert.Empty(val)
36
+
37
+
prev, err = tree.Insert([]byte("abc"), c3)
38
+
assert.NoError(err)
39
+
assert.NotEmpty(prev)
40
+
assert.Equal(&c2, prev)
41
+
42
+
val, err = tree.Get([]byte("abc"))
43
+
assert.NoError(err)
44
+
assert.Equal(&c3, val)
45
+
46
+
prev, err = tree.Insert([]byte("aaa"), c2)
47
+
assert.NoError(err)
48
+
assert.Empty(prev)
49
+
50
+
prev, err = tree.Insert([]byte("zzz"), c3)
51
+
assert.NoError(err)
52
+
assert.Empty(prev)
53
+
54
+
val, err = tree.Get([]byte("zzz"))
55
+
assert.NoError(err)
56
+
assert.Equal(&c3, val)
57
+
58
+
m := make(map[string]cid.Cid)
59
+
assert.NoError(tree.WriteToMap(m))
60
+
//fmt.Println("-----")
61
+
//debugPrintMap(m)
62
+
//fmt.Println("-----")
63
+
//debugPrintTree(tree, 0)
64
+
65
+
prev, err = tree.Remove([]byte("abc"))
66
+
assert.NoError(err)
67
+
assert.NotEmpty(prev)
68
+
assert.Equal(&c3, prev)
69
+
70
+
assert.NoError(tree.Verify())
71
+
72
+
}
73
+
74
+
func TestKeyLimits(t *testing.T) {
75
+
assert := assert.New(t)
76
+
77
+
var err error
78
+
tree := NewEmptyTree()
79
+
c2, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222")
80
+
81
+
emptyKey := []byte{}
82
+
_, err = tree.Get(emptyKey)
83
+
assert.Error(err)
84
+
_, err = tree.Remove(emptyKey)
85
+
assert.Error(err)
86
+
_, err = tree.Insert(emptyKey, c2)
87
+
assert.Error(err)
88
+
89
+
bigKey := bytes.Repeat([]byte{'a'}, 3000)
90
+
_, err = tree.Get(bigKey)
91
+
assert.Error(err)
92
+
_, err = tree.Remove(bigKey)
93
+
assert.Error(err)
94
+
_, err = tree.Insert(bigKey, c2)
95
+
assert.Error(err)
96
+
}
97
+
98
+
func TestBasicMap(t *testing.T) {
99
+
assert := assert.New(t)
100
+
101
+
c2, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222")
102
+
c3, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333")
103
+
assert.NotEmpty(c2)
104
+
assert.NotEmpty(c3)
105
+
106
+
inMap := map[string]cid.Cid{
107
+
"a": c2,
108
+
"b": c2,
109
+
"c": c2,
110
+
"d": c3,
111
+
"e": c3,
112
+
"f": c3,
113
+
"g": c3,
114
+
"h": c3,
115
+
"i": c3,
116
+
}
117
+
118
+
tree, err := LoadTreeFromMap(inMap)
119
+
assert.NoError(err)
120
+
121
+
//fmt.Println("-----")
122
+
//debugPrintTree(tree, 0)
123
+
assert.NoError(tree.Verify())
124
+
125
+
outMap := make(map[string]cid.Cid, len(inMap))
126
+
err = tree.WriteToMap(outMap)
127
+
assert.NoError(err)
128
+
assert.Equal(inMap, outMap)
129
+
}
130
+
131
+
func randomCid() cid.Cid {
132
+
buf := make([]byte, 32)
133
+
rand.Read(buf)
134
+
c, err := cid.NewPrefixV1(cid.Raw, multihash.SHA2_256).Sum(buf)
135
+
if err != nil {
136
+
panic(err)
137
+
}
138
+
return c
139
+
}
140
+
141
+
func randomStr() string {
142
+
buf := make([]byte, 16)
143
+
rand.Read(buf)
144
+
return hex.EncodeToString(buf)
145
+
}
146
+
147
+
func TestRandomTree(t *testing.T) {
148
+
assert := assert.New(t)
149
+
150
+
size := 200
151
+
152
+
inMap := make(map[string]cid.Cid, size)
153
+
outMap := make(map[string]cid.Cid, size)
154
+
155
+
for range size {
156
+
k := randomStr()
157
+
// ensure key is not already in the random set
158
+
for {
159
+
_, ok := inMap[k]
160
+
if !ok {
161
+
break
162
+
}
163
+
k = randomStr()
164
+
}
165
+
inMap[k] = randomCid()
166
+
}
167
+
168
+
tree, err := LoadTreeFromMap(inMap)
169
+
assert.NoError(err)
170
+
171
+
//fmt.Println("-----")
172
+
//debugPrintTree(tree, 0)
173
+
assert.NoError(tree.Verify())
174
+
assert.Equal(size, debugCountEntries(tree.Root))
175
+
176
+
err = tree.WriteToMap(outMap)
177
+
assert.NoError(err)
178
+
assert.Equal(len(inMap), len(outMap))
179
+
assert.Equal(inMap, outMap)
180
+
181
+
mapKeys := make([]string, len(inMap))
182
+
i := 0
183
+
for k, _ := range inMap {
184
+
mapKeys[i] = k
185
+
i++
186
+
}
187
+
rand.Shuffle(len(mapKeys), func(i, j int) {
188
+
mapKeys[i], mapKeys[j] = mapKeys[j], mapKeys[i]
189
+
})
190
+
191
+
// test gets
192
+
for _, k := range mapKeys {
193
+
val, err := tree.Get([]byte(k))
194
+
assert.NoError(err)
195
+
assert.Equal(inMap[k], *val)
196
+
}
197
+
198
+
// finally, removals
199
+
var val *cid.Cid
200
+
for _, k := range mapKeys {
201
+
val, err = tree.Remove([]byte(k))
202
+
assert.NoError(err)
203
+
assert.NotNil(val)
204
+
if err != nil {
205
+
break
206
+
}
207
+
err = tree.Verify()
208
+
assert.NoError(err)
209
+
if err != nil {
210
+
break
211
+
}
212
+
}
213
+
}
214
+
215
+
func TestRandomUntilError(t *testing.T) {
216
+
assert := assert.New(t)
217
+
var err error
218
+
var prev *cid.Cid
219
+
220
+
size := 200
221
+
222
+
tree := NewEmptyTree()
223
+
count := 0
224
+
//fmt.Println("-----")
225
+
for range size {
226
+
key := []byte(randomStr())
227
+
val := randomCid()
228
+
//fmt.Printf("%s %s\n", key, val)
229
+
prev, err = tree.Insert(key, val)
230
+
assert.NoError(err)
231
+
if prev == nil {
232
+
count++
233
+
}
234
+
235
+
assert.Equal(count, debugCountEntries(tree.Root))
236
+
err = tree.Verify()
237
+
assert.NoError(err)
238
+
if err != nil || count != debugCountEntries(tree.Root) {
239
+
//fmt.Println("-----")
240
+
//debugPrintTree(tree, 0)
241
+
break
242
+
}
243
+
}
244
+
}
245
+
246
+
func TestBrokenCaseOne(t *testing.T) {
247
+
assert := assert.New(t)
248
+
var err error
249
+
250
+
entries := [][]string{
251
+
{"1ea173efefa4", "bafkreibey6qzs7vb4wzlzfo7flflevl7qstzaggooiqivuexb6snapadq4"},
252
+
{"bed5c5789108", "bafkreifoxw552rsnuoargsfilhwmhprxr6qyzjmbtgjzmboii4x4mk4aoi"},
253
+
{"340b57a94d4c", "bafkreigarcm3fvnekjml6vmm5dyg46qnfkpc2lhghnh2wvntwvbrvxzq7q"},
254
+
{"8d37e30d3d29", "bafkreifdgiz7dmgng4aebiw5m6w4cypiiar2edgtkkfg47o3pniir3pxve"},
255
+
{"ee4b5efda333", "bafkreiho7qtewg7fm7egxe2ectkm2ykqygakph3nt4rrlp5mxwkvdwckk4"},
256
+
{"1180aeeadc01", "bafkreifqhtleufnxv2nkwehoa5lgmilwgkfqvlpkwbalvka6m6675ewkhu"},
257
+
{"c368b6b55998", "bafkreial4xepr5wnhetxnkmylmipdmjybxsgf74becdi74olmzb5w5gpiq"},
258
+
{"b948d2e0fc76", "bafkreiaefdmlyfjf4qovfyn22zbpw57wu667jtrvavogfxr7drewx4u24y"},
259
+
{"93c53d491ffd", "bafkreie2nxdmjsy6k6lendnsy7bzyufj7j37l42ymquwmpuzsauraqsibq"},
260
+
{"54ef0958a374", "bafkreigbnjxc7wbxgqxs2n2djjmlxnuf222gdiq4jgdtkse4yn67v5crq4"},
261
+
}
262
+
263
+
tree := NewEmptyTree()
264
+
for _, row := range entries {
265
+
val, _ := cid.Decode(row[1])
266
+
_, err = tree.Insert([]byte(row[0]), val)
267
+
assert.NoError(err)
268
+
}
269
+
270
+
//fmt.Println("-----")
271
+
//debugPrintNodePointers(tree)
272
+
//debugPrintChildPointers(tree)
273
+
//debugPrintTree(tree, 0)
274
+
assert.Equal(len(entries), debugCountEntries(tree.Root))
275
+
assert.NoError(tree.Verify())
276
+
}
+296
atproto/repo/mst/node.go
+296
atproto/repo/mst/node.go
···
1
+
package mst
2
+
3
+
import (
4
+
"bytes"
5
+
"fmt"
6
+
7
+
"github.com/ipfs/go-cid"
8
+
)
9
+
10
+
// Represents a node in a Merkle Search Tree (MST). If this is the "root" or "top" of the tree, it effectively is the tree itself.
11
+
//
12
+
// Trees may be "partial" if they contain references to child nodes by CID, but not pointers to `Node` representations.
13
+
type Node struct {
14
+
// array of key/value pairs and pointers to child nodes. entry arrays must always be in correct/valid order at any point in time: sorted by 'key', and at most one 'pointer' entry between 'value' entries.
15
+
Entries []NodeEntry
16
+
// "height" or "layer" of MST tree this node is at (with zero at the "bottom" and root/top of tree the "highest")
17
+
Height int
18
+
// if true, the cached CID of this node is out of date
19
+
Dirty bool
20
+
// optionally, the last computed CID of this Node (when expressed as NodeData)
21
+
CID *cid.Cid
22
+
// if true, this is an empty/incomplete node which just represents the CID of the tree. only used as part of MST inversion
23
+
Stub bool
24
+
}
25
+
26
+
// Represents an entry in an MST `Node`, which could either be a direct path/value entry, or a pointer do a child tree node. Note that these are *not* one-to-one with `EntryData`.
27
+
//
28
+
// Either the Key and Value fields should be non-zero; or the Child and/or ChildCID field should be non-zero.
29
+
// If ChildCID is present, but Child is not, then this is part of a "partial" tree.
30
+
type NodeEntry struct {
31
+
Key []byte
32
+
Value *cid.Cid
33
+
ChildCID *cid.Cid
34
+
Child *Node
35
+
36
+
// tracks whether anything about this entry has changed since `Node` CID was computed
37
+
Dirty bool
38
+
}
39
+
40
+
func (n *Node) IsEmpty() bool {
41
+
return len(n.Entries) == 0
42
+
}
43
+
44
+
// Checks if the sub-tree (this node, or any children, recursively) contains any CID references to nodes which are not present.
45
+
func (n *Node) IsPartial() bool {
46
+
if n.Stub {
47
+
return true
48
+
}
49
+
for _, e := range n.Entries {
50
+
if e.ChildCID != nil && e.Child == nil {
51
+
return true
52
+
}
53
+
if e.Child != nil && e.Child.IsPartial() {
54
+
return true
55
+
}
56
+
}
57
+
return false
58
+
}
59
+
60
+
// Returns true if this entry is a key/value at the current node
61
+
func (e *NodeEntry) IsValue() bool {
62
+
if len(e.Key) > 0 && e.Value != nil {
63
+
return true
64
+
}
65
+
return false
66
+
}
67
+
68
+
// Returns true if this entry points to a node on a lower level
69
+
func (e *NodeEntry) IsChild() bool {
70
+
if e.Child != nil || e.ChildCID != nil {
71
+
return true
72
+
}
73
+
return false
74
+
}
75
+
76
+
// creates a deep/recursive copy of the sub-tree
77
+
func (n *Node) deepCopy() *Node {
78
+
out := Node{
79
+
Entries: make([]NodeEntry, len(n.Entries)),
80
+
Height: n.Height,
81
+
Dirty: n.Dirty,
82
+
Stub: n.Stub,
83
+
CID: n.CID,
84
+
}
85
+
for i, e := range n.Entries {
86
+
out.Entries[i] = NodeEntry{
87
+
Key: e.Key,
88
+
Value: e.Value,
89
+
ChildCID: e.ChildCID,
90
+
Dirty: e.Dirty,
91
+
}
92
+
if e.Child != nil {
93
+
out.Entries[i].Child = e.Child.deepCopy()
94
+
}
95
+
}
96
+
return &out
97
+
}
98
+
99
+
// Looks for a "value" entry in the node with the exact key.
100
+
// Returns entry index if a matching entry is found; or -1 if not found
101
+
func (n *Node) findExistingEntry(key []byte) int {
102
+
for i, e := range n.Entries {
103
+
// TODO perf: could skip early if e.Key is lower
104
+
if e.IsValue() && bytes.Equal(key, e.Key) {
105
+
return i
106
+
}
107
+
}
108
+
return -1
109
+
}
110
+
111
+
// Looks for a "child" entry which the key would live under.
112
+
//
113
+
// Returns -1 if not found.
114
+
func (n *Node) findExistingChild(key []byte) int {
115
+
idx := -1
116
+
for i, e := range n.Entries {
117
+
if e.IsChild() {
118
+
idx = i
119
+
continue
120
+
}
121
+
if e.IsValue() {
122
+
if bytes.Compare(key, e.Key) <= 0 {
123
+
break
124
+
}
125
+
idx = -1
126
+
}
127
+
}
128
+
return idx
129
+
}
130
+
131
+
// Determines index where a new entry (child or value) would be inserted, relevant to the given key.
132
+
//
133
+
// If the key would "split" an existing child entry, the index of that entry is returned, and a flag set
134
+
//
135
+
// If the entry would be appended, then the index returned will be one higher that the current largest index.
136
+
func (n *Node) findInsertionIndex(key []byte) (idx int, split bool, retErr error) {
137
+
if n.Stub {
138
+
return -1, false, fmt.Errorf("partial MST, can't determine insertion order")
139
+
}
140
+
for i, e := range n.Entries {
141
+
if e.IsValue() {
142
+
if bytes.Compare(key, e.Key) < 0 {
143
+
return i, false, nil
144
+
}
145
+
}
146
+
if e.IsChild() {
147
+
// first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child
148
+
if i+1 < len(n.Entries) {
149
+
next := n.Entries[i+1]
150
+
if next.IsValue() && bytes.Compare(key, next.Key) > 0 {
151
+
continue
152
+
}
153
+
}
154
+
if e.Child == nil {
155
+
return -1, false, fmt.Errorf("partial MST, can't determine insertion order")
156
+
}
157
+
order, err := e.Child.compareKey(key, false)
158
+
if err != nil {
159
+
return -1, false, err
160
+
}
161
+
if order < 0 {
162
+
// key comes before this entire child sub-tree
163
+
return i, false, nil
164
+
}
165
+
if order > 0 {
166
+
// key comes after this entire child sub-tree
167
+
continue
168
+
}
169
+
// key falls inside this child sub-tree
170
+
return i, true, nil
171
+
}
172
+
}
173
+
174
+
// would need to be appended after
175
+
return len(n.Entries), false, nil
176
+
}
177
+
178
+
// Compares a provided `key` against the overall range of keys represented by a `Node`. Returns -1 if the key sorts lower than all keys (recursively) covered by the Node; 1 if higher, and 0 if the key falls within Node's key range.
179
+
//
180
+
// If the `markDirty` flag is true, then this method will set the Dirty flag on this node, and any child nodes which were needed to "prove" the key order. This can be used to mark nodes for inclusion in invertible MST diffs.
181
+
func (n *Node) compareKey(key []byte, markDirty bool) (int, error) {
182
+
if n.Stub {
183
+
return -1, ErrPartialTree
184
+
}
185
+
if n.IsEmpty() {
186
+
// TODO: should we actually return 0 in this case?
187
+
return 0, fmt.Errorf("can't determine key range of empty MST node")
188
+
}
189
+
if markDirty == true {
190
+
n.Dirty = true
191
+
}
192
+
// check if lower than this entire node
193
+
e := n.Entries[0]
194
+
if e.IsValue() && bytes.Compare(key, e.Key) < 0 {
195
+
return -1, nil
196
+
}
197
+
// check if higher than this entire node
198
+
e = n.Entries[len(n.Entries)-1]
199
+
if e.IsValue() && bytes.Compare(key, e.Key) > 0 {
200
+
return 1, nil
201
+
}
202
+
for i, e := range n.Entries {
203
+
if e.IsValue() && bytes.Compare(key, e.Key) < 0 {
204
+
// we don't need to recurse/iterate further
205
+
return 0, nil
206
+
}
207
+
if e.IsChild() {
208
+
// first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child
209
+
if i+1 < len(n.Entries) {
210
+
next := n.Entries[i+1]
211
+
if next.IsValue() && bytes.Compare(key, next.Key) > 0 {
212
+
continue
213
+
}
214
+
}
215
+
if e.Child == nil {
216
+
return 0, fmt.Errorf("%w: can't compare key order recursively", ErrPartialTree)
217
+
}
218
+
order, err := e.Child.compareKey(key, markDirty)
219
+
if err != nil {
220
+
return 0, err
221
+
}
222
+
// lower than entire node
223
+
if i == 0 && order < 0 {
224
+
return -1, nil
225
+
}
226
+
// higher than entire node
227
+
if i == len(n.Entries)-1 && order > 0 {
228
+
return 1, nil
229
+
}
230
+
return 0, nil
231
+
}
232
+
}
233
+
return 0, nil
234
+
}
235
+
236
+
// helper function, mostly for testing or development, which redusively inserts key/CID pairs into a `map[string]cid.Cid
237
+
func (n *Node) writeToMap(m map[string]cid.Cid) error {
238
+
if m == nil {
239
+
return fmt.Errorf("un-initialized map as an argument")
240
+
}
241
+
if n == nil {
242
+
return fmt.Errorf("nil tree pointer")
243
+
}
244
+
for _, e := range n.Entries {
245
+
if e.IsValue() {
246
+
m[string(e.Key)] = *e.Value
247
+
}
248
+
if e.Child != nil {
249
+
if err := e.Child.writeToMap(m); err != nil {
250
+
return fmt.Errorf("failed to export MST structure as map: %w", err)
251
+
}
252
+
}
253
+
}
254
+
return nil
255
+
}
256
+
257
+
// Reads the value (CID) corresponding to the key. If key is not in the tree, returns (nil, nil).
258
+
//
259
+
// n: Node at top of sub-tree to operate on. Must not be nil.
260
+
// key: key or path being inserted. must not be empty/nil
261
+
// height: tree height corresponding to key. if a negative value is provided, will be computed; use -1 instead of 0 if height is not known
262
+
func (n *Node) getCID(key []byte, height int) (*cid.Cid, error) {
263
+
if n.Stub {
264
+
return nil, ErrPartialTree
265
+
}
266
+
if height < 0 {
267
+
height = HeightForKey(key)
268
+
}
269
+
270
+
if height > n.Height {
271
+
// key from a higher layer; key was not in tree
272
+
return nil, nil
273
+
}
274
+
275
+
if height < n.Height {
276
+
// look for a child node
277
+
idx := n.findExistingChild(key)
278
+
if idx >= 0 {
279
+
if n.Entries[idx].Child == nil {
280
+
return nil, fmt.Errorf("could not search for key: %w", ErrPartialTree)
281
+
}
282
+
return n.Entries[idx].Child.getCID(key, height)
283
+
}
284
+
// otherwise, not found
285
+
return nil, nil
286
+
}
287
+
288
+
// search at this height
289
+
idx := n.findExistingEntry(key)
290
+
if idx >= 0 {
291
+
return n.Entries[idx].Value, nil
292
+
}
293
+
294
+
// not found
295
+
return nil, nil
296
+
}
+230
atproto/repo/mst/node_insert.go
+230
atproto/repo/mst/node_insert.go
···
1
+
package mst
2
+
3
+
import (
4
+
"fmt"
5
+
"slices"
6
+
7
+
"github.com/ipfs/go-cid"
8
+
)
9
+
10
+
// Adds a key/CID entry to a sub-tree defined by a Node. If a previous value existed, returns it.
11
+
//
12
+
// If the insert is a no-op (the key already existed with exact value), then the operation is a no-op, the tree is not marked dirty, and the val is returned as the 'prev' value.
13
+
//
14
+
// n: Node at top of sub-tree to operate on
15
+
// key: key or path being inserted. must not be empty/nil
16
+
// val: CID value being inserted
17
+
// height: tree height to insert at, derived from key. if a negative value is provided, will be computed; use -1 instead of 0 if height is not known
18
+
func (n *Node) insert(key []byte, val cid.Cid, height int) (*Node, *cid.Cid, error) {
19
+
if n.Stub {
20
+
return nil, nil, ErrPartialTree
21
+
}
22
+
if height < 0 {
23
+
height = HeightForKey(key)
24
+
}
25
+
26
+
if n == nil {
27
+
return nil, nil, fmt.Errorf("operating on nil tree/node")
28
+
}
29
+
30
+
for height > n.Height {
31
+
// if the new key is higher in the tree; will need to add a parent node, which may involve splitting this current node
32
+
return n.insertParent(key, val, height)
33
+
}
34
+
35
+
// if key is lower on the tree, we need to descend first
36
+
if height < n.Height {
37
+
return n.insertChild(key, val, height)
38
+
}
39
+
40
+
// look for existing key
41
+
idx := n.findExistingEntry(key)
42
+
if idx >= 0 {
43
+
e := n.Entries[idx]
44
+
if *e.Value == val {
45
+
// same value already exists; no-op
46
+
return n, &val, nil
47
+
}
48
+
// update operation
49
+
prev := e.Value
50
+
n.Entries[idx].Value = &val
51
+
n.Entries[idx].Dirty = true
52
+
n.Dirty = true
53
+
return n, prev, nil
54
+
}
55
+
56
+
// insert new entry to this node
57
+
idx, split, err := n.findInsertionIndex(key)
58
+
if err != nil {
59
+
return nil, nil, err
60
+
}
61
+
n.Dirty = true
62
+
newEntry := NodeEntry{
63
+
Key: key,
64
+
Value: &val,
65
+
Dirty: true,
66
+
}
67
+
68
+
if !split {
69
+
// TODO: is this really necessary? or can we just slices.Insert beyond the end of a slice?
70
+
if idx >= len(n.Entries) {
71
+
n.Entries = append(n.Entries, newEntry)
72
+
} else {
73
+
n.Entries = slices.Insert(n.Entries, idx, newEntry)
74
+
}
75
+
return n, nil, nil
76
+
}
77
+
78
+
// we need to split
79
+
e := n.Entries[idx]
80
+
left, right, err := e.Child.split(key)
81
+
if err != nil {
82
+
return nil, nil, err
83
+
}
84
+
// remove the existing entry, and replace with three new entries
85
+
n.Entries = slices.Delete(n.Entries, idx, idx+1)
86
+
n.Entries = slices.Insert(
87
+
n.Entries,
88
+
idx,
89
+
NodeEntry{Child: left, Dirty: true},
90
+
newEntry,
91
+
NodeEntry{Child: right, Dirty: true},
92
+
)
93
+
return n, nil, nil
94
+
}
95
+
96
+
func (n *Node) splitEntries(idx int) (*Node, *Node, error) {
97
+
if idx == 0 || idx >= len(n.Entries) {
98
+
return nil, nil, fmt.Errorf("splitting at one end or the other of entries")
99
+
}
100
+
left := Node{
101
+
Height: n.Height,
102
+
Dirty: true,
103
+
Entries: n.Entries[:idx],
104
+
}
105
+
right := Node{
106
+
Height: n.Height,
107
+
Dirty: true,
108
+
// don't use the same slice here
109
+
Entries: append([]NodeEntry{}, n.Entries[idx:]...),
110
+
}
111
+
if left.IsEmpty() || right.IsEmpty() {
112
+
return nil, nil, fmt.Errorf("one of the legs is empty (idx=%d, len=%d)", idx, len(n.Entries))
113
+
}
114
+
return &left, &right, nil
115
+
}
116
+
117
+
func (n *Node) split(key []byte) (*Node, *Node, error) {
118
+
if n.IsEmpty() {
119
+
// TODO: this feels defensive and could be removed
120
+
return nil, nil, fmt.Errorf("tried to split an empty node")
121
+
}
122
+
123
+
idx, split, err := n.findInsertionIndex(key)
124
+
if err != nil {
125
+
return nil, nil, err
126
+
}
127
+
if !split {
128
+
// simple split based on values
129
+
return n.splitEntries(idx)
130
+
}
131
+
132
+
// need to split recursively
133
+
e := n.Entries[idx]
134
+
lowerLeft, lowerRight, err := e.Child.split(key)
135
+
if err != nil {
136
+
return nil, nil, err
137
+
}
138
+
left := &Node{
139
+
Height: n.Height,
140
+
Dirty: true,
141
+
Entries: []NodeEntry{},
142
+
}
143
+
left.Entries = append(left.Entries, n.Entries[:idx]...)
144
+
left.Entries = append(left.Entries, NodeEntry{Child: lowerLeft, Dirty: true})
145
+
right := &Node{
146
+
Height: n.Height,
147
+
Dirty: true,
148
+
Entries: []NodeEntry{NodeEntry{Child: lowerRight, Dirty: true}},
149
+
}
150
+
if idx+1 < len(n.Entries) {
151
+
right.Entries = append(right.Entries, n.Entries[idx+1:]...)
152
+
}
153
+
return left, right, nil
154
+
}
155
+
156
+
// inserts a node "above" this node in tree, possibly splitting the current node
157
+
func (n *Node) insertParent(key []byte, val cid.Cid, height int) (*Node, *cid.Cid, error) {
158
+
var parent *Node
159
+
if n.IsEmpty() {
160
+
// if current node is empty, just replace directly with current height
161
+
parent = &Node{
162
+
Height: height,
163
+
Dirty: true,
164
+
}
165
+
} else {
166
+
// otherwise push a layer and recurse
167
+
parent = &Node{
168
+
Height: n.Height + 1,
169
+
Dirty: true,
170
+
Entries: []NodeEntry{NodeEntry{
171
+
Child: n,
172
+
Dirty: true,
173
+
}},
174
+
}
175
+
}
176
+
// regular insertion will handle any necessary "split"
177
+
return parent.insert(key, val, height)
178
+
}
179
+
180
+
// inserts a node "below" this node in tree; either creating a new child entry or re-using an existing one
181
+
func (n *Node) insertChild(key []byte, val cid.Cid, height int) (*Node, *cid.Cid, error) {
182
+
// look for an existing child node which encompasses the key, and use that
183
+
idx := n.findExistingChild(key)
184
+
if idx >= 0 {
185
+
e := n.Entries[idx]
186
+
if e.Child == nil {
187
+
return nil, nil, fmt.Errorf("could not insert key: %w", ErrPartialTree)
188
+
}
189
+
newChild, prev, err := e.Child.insert(key, val, height)
190
+
if err != nil {
191
+
return nil, nil, err
192
+
}
193
+
if prev != nil && *prev == val {
194
+
// no-op
195
+
return n, &val, nil
196
+
}
197
+
n.Dirty = true
198
+
n.Entries[idx].Child = newChild
199
+
n.Entries[idx].Dirty = true
200
+
return n, prev, nil
201
+
}
202
+
203
+
// insert a new child node. this might be recursive if the child is not a *direct* child
204
+
idx, split, err := n.findInsertionIndex(key)
205
+
if err != nil {
206
+
return nil, nil, err
207
+
}
208
+
if split {
209
+
return nil, nil, fmt.Errorf("unexpected split when inserting child")
210
+
}
211
+
n.Dirty = true
212
+
newChild := &Node{
213
+
Height: n.Height - 1,
214
+
Dirty: true,
215
+
}
216
+
newChild, _, err = newChild.insert(key, val, height)
217
+
if err != nil {
218
+
return nil, nil, err
219
+
}
220
+
newEntry := NodeEntry{
221
+
Child: newChild,
222
+
Dirty: true,
223
+
}
224
+
if idx == len(n.Entries) {
225
+
n.Entries = append(n.Entries, newEntry)
226
+
} else {
227
+
n.Entries = slices.Insert(n.Entries, idx, newEntry)
228
+
}
229
+
return n, nil, nil
230
+
}
+187
atproto/repo/mst/node_remove.go
+187
atproto/repo/mst/node_remove.go
···
1
+
package mst
2
+
3
+
import (
4
+
"bytes"
5
+
"fmt"
6
+
"slices"
7
+
8
+
"github.com/ipfs/go-cid"
9
+
)
10
+
11
+
// Removes key/value from the sub-tree provided, returning a new tree, and the previous CID value. If key is not found, returns unmodified subtree, and nil for the returned CID.
12
+
//
13
+
// n: Node at top of sub-tree to operate on. Must not be nil.
14
+
// key: key or path being inserted. must not be empty/nil
15
+
// height: tree height corresponding to key. if a negative value is provided, will be computed; use -1 instead of 0 if height is not known
16
+
func (n *Node) remove(key []byte, height int) (*Node, *cid.Cid, error) {
17
+
if n.Stub {
18
+
return nil, nil, ErrPartialTree
19
+
}
20
+
// TODO: do we need better handling of "is this the top"?
21
+
top := false
22
+
if height < 0 {
23
+
top = true
24
+
height = HeightForKey(key)
25
+
}
26
+
27
+
if height > n.Height {
28
+
// removing a key from a higher layer; key was not in tree
29
+
return n, nil, nil
30
+
}
31
+
32
+
if height < n.Height {
33
+
// TODO: handle case of this returning an empty node at top of tree, with wrong height
34
+
return n.removeChild(key, height)
35
+
}
36
+
37
+
// look at this level
38
+
idx := n.findExistingEntry(key)
39
+
if idx < 0 {
40
+
// key not found
41
+
return n, nil, nil
42
+
}
43
+
44
+
// found it! will remove from list
45
+
n.Dirty = true
46
+
prev := n.Entries[idx].Value
47
+
48
+
// check if we need to "merge" adjacent nodes
49
+
if idx > 0 && idx+1 < len(n.Entries) && n.Entries[idx-1].IsChild() && n.Entries[idx+1].IsChild() {
50
+
if n.Entries[idx-1].Child == nil || n.Entries[idx+1].Child == nil {
51
+
return nil, nil, fmt.Errorf("can not merge child nodes: %w", ErrPartialTree)
52
+
}
53
+
newChild, err := mergeNodes(n.Entries[idx-1].Child, n.Entries[idx+1].Child)
54
+
if err != nil {
55
+
return nil, nil, err
56
+
}
57
+
n.Entries = slices.Delete(n.Entries, idx, idx+2)
58
+
n.Entries[idx-1] = NodeEntry{Child: newChild, Dirty: true}
59
+
} else {
60
+
// simple removal
61
+
n.Entries = slices.Delete(n.Entries, idx, idx+1)
62
+
}
63
+
64
+
// marks adjacent child nodes dirty to include as "proof"
65
+
proveDeletion(n, key)
66
+
67
+
// check if top of node is now just a pointer
68
+
if top {
69
+
for {
70
+
if len(n.Entries) != 1 || !n.Entries[0].IsChild() {
71
+
break
72
+
}
73
+
if n.Entries[0].Child == nil {
74
+
// this is something of a hack, for MST inversion which requires trimming the tree
75
+
if n.Entries[0].ChildCID == nil {
76
+
return nil, nil, fmt.Errorf("can not prune top of tree: %w", ErrPartialTree)
77
+
} else {
78
+
n = &Node{
79
+
Height: n.Height - 1,
80
+
Stub: true,
81
+
CID: n.Entries[0].ChildCID,
82
+
}
83
+
}
84
+
} else {
85
+
n = n.Entries[0].Child
86
+
}
87
+
}
88
+
}
89
+
return n, prev, nil
90
+
}
91
+
92
+
func proveDeletion(n *Node, key []byte) error {
93
+
for i, e := range n.Entries {
94
+
if e.IsValue() {
95
+
if bytes.Compare(key, e.Key) < 0 {
96
+
return nil
97
+
}
98
+
}
99
+
if e.IsChild() {
100
+
// first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child
101
+
if i+1 < len(n.Entries) {
102
+
next := n.Entries[i+1]
103
+
if next.IsValue() && bytes.Compare(key, next.Key) > 0 {
104
+
continue
105
+
}
106
+
}
107
+
if e.Child == nil {
108
+
return fmt.Errorf("can't prove deletion: %w", ErrPartialTree)
109
+
}
110
+
order, err := e.Child.compareKey(key, true)
111
+
if err != nil {
112
+
return err
113
+
}
114
+
if order > 0 {
115
+
// key comes after this entire child sub-tree
116
+
continue
117
+
}
118
+
if order < 0 {
119
+
return nil
120
+
}
121
+
// key falls inside this child sub-tree
122
+
return proveDeletion(e.Child, key)
123
+
}
124
+
}
125
+
return nil
126
+
}
127
+
128
+
func mergeNodes(left *Node, right *Node) (*Node, error) {
129
+
idx := len(left.Entries)
130
+
n := &Node{
131
+
Height: left.Height,
132
+
Dirty: true,
133
+
Entries: append(left.Entries, right.Entries...),
134
+
}
135
+
if n.Entries[idx-1].IsChild() && n.Entries[idx].IsChild() {
136
+
// need to merge recursively
137
+
lowerLeft := n.Entries[idx-1]
138
+
lowerRight := n.Entries[idx]
139
+
if lowerLeft.Child == nil || lowerRight.Child == nil {
140
+
return nil, fmt.Errorf("can not merge child nodes: %w", ErrPartialTree)
141
+
}
142
+
lowerMerged, err := mergeNodes(lowerLeft.Child, lowerRight.Child)
143
+
if err != nil {
144
+
return nil, err
145
+
}
146
+
n.Entries[idx-1] = NodeEntry{Child: lowerMerged, Dirty: true}
147
+
n.Entries = slices.Delete(n.Entries, idx, idx+1)
148
+
}
149
+
return n, nil
150
+
}
151
+
152
+
// internal helper
153
+
func (n *Node) removeChild(key []byte, height int) (*Node, *cid.Cid, error) {
154
+
// look for a child
155
+
idx := n.findExistingChild(key)
156
+
if idx < 0 {
157
+
// no child pointer; key not in tree
158
+
return n, nil, nil
159
+
}
160
+
161
+
e := n.Entries[idx]
162
+
if e.Child == nil {
163
+
// partial node, can't recurse
164
+
return nil, nil, fmt.Errorf("could not remove key: %w", ErrPartialTree)
165
+
}
166
+
newChild, prev, err := e.Child.remove(key, height)
167
+
if err != nil {
168
+
return nil, nil, err
169
+
}
170
+
if prev == nil {
171
+
// no-op
172
+
return n, nil, nil
173
+
}
174
+
175
+
// if the child node was updated, but still exists, just update pointer
176
+
if !newChild.IsEmpty() {
177
+
n.Dirty = true
178
+
n.Entries[idx].Child = newChild
179
+
n.Entries[idx].Dirty = true
180
+
return n, prev, nil
181
+
}
182
+
183
+
// if new child was empty, remove it from entry list; note that *this* entry might now be empty
184
+
n.Dirty = true
185
+
n.Entries = slices.Delete(n.Entries, idx, idx+1)
186
+
return n, prev, nil
187
+
}
+160
atproto/repo/mst/tree.go
+160
atproto/repo/mst/tree.go
···
1
+
package mst
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"fmt"
7
+
8
+
"github.com/ipfs/go-cid"
9
+
blockstore "github.com/ipfs/go-ipfs-blockstore"
10
+
)
11
+
12
+
// High-level API for an MST, as a decoded in-memory data structure.
13
+
//
14
+
// This might be an entire tree (all child nodes in-memory), or might be a partial tree with some nodes as CID links. Operations on the tree do not persist to any backing storage automatically.
15
+
//
16
+
// Errors when operating on the tree may leave the tree in a partially modified or invalid/corrupt state.
17
+
type Tree struct {
18
+
Root *Node
19
+
// TODO: have a blockstore.Blockstore for loading lazily?
20
+
}
21
+
22
+
var ErrInvalidKey = errors.New("bytestring not a valid MST key")
23
+
24
+
var ErrPartialTree = errors.New("MST is not complete")
25
+
26
+
var ErrInvalidTree = errors.New("invalid MST structure")
27
+
28
+
func NewEmptyTree() Tree {
29
+
return Tree{
30
+
Root: &Node{
31
+
Dirty: true,
32
+
Height: 0,
33
+
},
34
+
}
35
+
}
36
+
37
+
// Adds a key/value to the tree, and returns any previously existing value (CID).
38
+
//
39
+
// Caller can inspect the previous value to determine if the behavior was a "creation" (key didn't exist), an "update" (key existed with a different value), or no-op (key existed with current value).
40
+
//
41
+
// key: key or path being inserted. must not be empty/nil
42
+
// val: CID value being inserted
43
+
func (t *Tree) Insert(key []byte, val cid.Cid) (*cid.Cid, error) {
44
+
if !IsValidKey(key) {
45
+
return nil, ErrInvalidKey
46
+
}
47
+
out, prev, err := t.Root.insert(key, val, -1)
48
+
if err != nil {
49
+
return nil, err
50
+
}
51
+
t.Root = out
52
+
return prev, nil
53
+
}
54
+
55
+
// Removes key/value from the sub-tree provided. Return the previous CID value, if any. If key was not found, returns nil (which is not an error).
56
+
//
57
+
// key: key or path being inserted. must not be empty/nil
58
+
func (t *Tree) Remove(key []byte) (*cid.Cid, error) {
59
+
if !IsValidKey(key) {
60
+
return nil, ErrInvalidKey
61
+
}
62
+
out, prev, err := t.Root.remove(key, -1)
63
+
if err != nil {
64
+
return nil, err
65
+
}
66
+
t.Root = out
67
+
return prev, nil
68
+
}
69
+
70
+
// Reads the value (CID) corresponding to the key.
71
+
//
72
+
// If key is not in the tree, returns nil, not an error.
73
+
//
74
+
// key: key or path being inserted. must not be empty/nil
75
+
func (t *Tree) Get(key []byte) (*cid.Cid, error) {
76
+
if !IsValidKey(key) {
77
+
return nil, ErrInvalidKey
78
+
}
79
+
return t.Root.getCID(key, -1)
80
+
}
81
+
82
+
// Creates a new Tree by loading key/value pairs from a map.
83
+
func LoadTreeFromMap(m map[string]cid.Cid) (*Tree, error) {
84
+
if m == nil {
85
+
return nil, fmt.Errorf("un-initialized map as an argument")
86
+
}
87
+
t := NewEmptyTree()
88
+
var err error
89
+
for key, val := range m {
90
+
_, err = t.Insert([]byte(key), val)
91
+
if err != nil {
92
+
return nil, fmt.Errorf("unexpected failure to build MST structure: %w", err)
93
+
}
94
+
}
95
+
return &t, nil
96
+
}
97
+
98
+
// Recursively walks the tree and writes key/value pairs to map `m`
99
+
//
100
+
// The map (`m`) is mutated in place (by reference); the map must be initialized before calling.
101
+
func (t *Tree) WriteToMap(m map[string]cid.Cid) error {
102
+
if m == nil {
103
+
return fmt.Errorf("un-initialized map as an argument")
104
+
}
105
+
if t.Root == nil {
106
+
return fmt.Errorf("empty tree root")
107
+
}
108
+
return t.Root.writeToMap(m)
109
+
}
110
+
111
+
// Returns the overall root-node CID for the MST.
112
+
//
113
+
// If possible, lazily returned a known value. If necessary, recursively encodes tree nodes to compute CIDs.
114
+
//
115
+
// NOTE: will mark the tree "clean" (clear any dirty flags).
116
+
func (t *Tree) RootCID() (*cid.Cid, error) {
117
+
if t.Root != nil && t.Root.Stub && !t.Root.Dirty && t.Root.CID != nil {
118
+
return t.Root.CID, nil
119
+
}
120
+
return t.Root.writeBlocks(context.Background(), nil, true)
121
+
}
122
+
123
+
// If the tree contains no key/value pairs, returns true.
124
+
func (t *Tree) IsEmpty() bool {
125
+
if t.Root == nil {
126
+
return true
127
+
}
128
+
return t.Root.IsEmpty()
129
+
}
130
+
131
+
// Returns false if all nodes in the tree are available in-memory in decoded format; otherwise returns true. Does not consider record data, only MST nodes.
132
+
func (t *Tree) IsPartial() bool {
133
+
if t.Root == nil {
134
+
return true
135
+
}
136
+
return t.Root.IsPartial()
137
+
}
138
+
139
+
// Creates a deep copy of MST
140
+
func (t *Tree) Copy() Tree {
141
+
return Tree{
142
+
Root: t.Root.deepCopy(),
143
+
}
144
+
}
145
+
146
+
func LoadTreeFromStore(ctx context.Context, bs blockstore.Blockstore, root cid.Cid) (*Tree, error) {
147
+
n, err := loadNodeFromStore(ctx, bs, root)
148
+
if err != nil {
149
+
return nil, err
150
+
}
151
+
n.ensureHeights()
152
+
return &Tree{
153
+
Root: n,
154
+
}, nil
155
+
}
156
+
157
+
// Walks the tree, encodes any "dirty" nodes as CBOR data, and writes that data as blocks to the provided blockstore. Returns root CID.
158
+
func (t *Tree) WriteDiffBlocks(ctx context.Context, bs blockstore.Blockstore) (*cid.Cid, error) {
159
+
return t.Root.writeBlocks(ctx, bs, true)
160
+
}
+57
atproto/repo/mst/util.go
+57
atproto/repo/mst/util.go
···
1
+
package mst
2
+
3
+
import (
4
+
"crypto/sha256"
5
+
)
6
+
7
+
const (
8
+
// Maximum length, in bytes, of a key in the tree. Note that the atproto specifications imply a repo path maximum length, but don't say anything directly about MST key, other than they can not be empty (zero-length).
9
+
MAX_KEY_BYTES = 1024
10
+
)
11
+
12
+
// Computes the MST "height" for a key (bytestring). Layers are counted from the "bottom" of the tree, starting with zero.
13
+
//
14
+
// For atproto repository v3, uses SHA-256 as the hashing function and counts two bits at a time, for an MST "fanout" value of 16.
15
+
func HeightForKey(key []byte) (height int) {
16
+
hv := sha256.Sum256(key)
17
+
for _, b := range hv {
18
+
if b&0xC0 != 0 {
19
+
// Common case. No leading pair of zero bits.
20
+
break
21
+
}
22
+
if b == 0x00 {
23
+
height += 4
24
+
continue
25
+
}
26
+
if b&0xFC == 0x00 {
27
+
height += 3
28
+
} else if b&0xF0 == 0x00 {
29
+
height += 2
30
+
} else {
31
+
height += 1
32
+
}
33
+
break
34
+
}
35
+
return height
36
+
}
37
+
38
+
// Computes the common prefix length between two bytestrings.
39
+
//
40
+
// Used when compacting node entry lists for encoding.
41
+
func CountPrefixLen(a, b []byte) int {
42
+
// This pattern avoids panicindex calls, as the Go compiler's prove pass can convince itself that neither a[i] nor b[i] are ever out of bounds.
43
+
var i int
44
+
for i = 0; i < len(a) && i < len(b); i++ {
45
+
if a[i] != b[i] {
46
+
return i
47
+
}
48
+
}
49
+
return i
50
+
}
51
+
52
+
func IsValidKey(key []byte) bool {
53
+
if len(key) == 0 || len(key) > MAX_KEY_BYTES {
54
+
return false
55
+
}
56
+
return true
57
+
}
+78
atproto/repo/mst/util_interop_test.go
+78
atproto/repo/mst/util_interop_test.go
···
1
+
package mst
2
+
3
+
import (
4
+
"testing"
5
+
6
+
"github.com/stretchr/testify/assert"
7
+
)
8
+
9
+
func TestPrefixLen(t *testing.T) {
10
+
msg := "length of common prefix between strings"
11
+
12
+
testVec := []struct {
13
+
Left []byte
14
+
Right []byte
15
+
Len int
16
+
}{
17
+
{[]byte(""), []byte(""), 0},
18
+
{[]byte("abc"), []byte("abc"), 3},
19
+
{[]byte(""), []byte("abc"), 0},
20
+
{[]byte("abc"), []byte(""), 0},
21
+
{[]byte("ab"), []byte("abc"), 2},
22
+
{[]byte("abc"), []byte("ab"), 2},
23
+
{[]byte("abcde"), []byte("abc"), 3},
24
+
{[]byte("abc"), []byte("abcde"), 3},
25
+
{[]byte("abcde"), []byte("abc1"), 3},
26
+
{[]byte("abcde"), []byte("abb"), 2},
27
+
{[]byte("abcde"), []byte("qbb"), 0},
28
+
{[]byte("abc"), []byte("abc\x00"), 3},
29
+
{[]byte("abc\x00"), []byte("abc"), 3},
30
+
}
31
+
32
+
for _, c := range testVec {
33
+
assert.Equal(t, c.Len, CountPrefixLen(c.Left, c.Right), msg)
34
+
}
35
+
}
36
+
37
+
func TestPrefixLenWide(t *testing.T) {
38
+
// NOTE: these are not cross-language consistent!
39
+
msg := "length of common prefix between strings (wide chars)"
40
+
41
+
assert.Equal(t, 9, len("jalapeño"), msg) // 8 in javascript
42
+
assert.Equal(t, 4, len("💩"), msg) // 2 in javascript
43
+
assert.Equal(t, 18, len("👩👧👧"), msg) // 8 in javascript
44
+
45
+
testVec := []struct {
46
+
Left []byte
47
+
Right []byte
48
+
Len int
49
+
}{
50
+
{[]byte(""), []byte(""), 0},
51
+
{[]byte("jalapeño"), []byte("jalapeno"), 6},
52
+
{[]byte("jalapeñoA"), []byte("jalapeñoB"), 9},
53
+
{[]byte("coöperative"), []byte("coüperative"), 3},
54
+
{[]byte("abc💩abc"), []byte("abcabc"), 3},
55
+
{[]byte("💩abc"), []byte("💩ab"), 6},
56
+
{[]byte("abc👩👦👦de"), []byte("abc👩👧👧de"), 13},
57
+
}
58
+
59
+
for _, c := range testVec {
60
+
assert.Equal(t, c.Len, CountPrefixLen(c.Left, c.Right), msg)
61
+
}
62
+
}
63
+
64
+
func TestHeightForKey(t *testing.T) {
65
+
assert := assert.New(t)
66
+
msg := "MST 'depth' computation (SHA-256 leading zeros)"
67
+
assert.Equal(HeightForKey([]byte("")), 0, msg)
68
+
assert.Equal(HeightForKey([]byte("asdf")), 0, msg)
69
+
assert.Equal(HeightForKey([]byte("blue")), 1, msg)
70
+
assert.Equal(HeightForKey([]byte("2653ae71")), 0, msg)
71
+
assert.Equal(HeightForKey([]byte("88bfafc7")), 2, msg)
72
+
assert.Equal(HeightForKey([]byte("2a92d355")), 4, msg)
73
+
assert.Equal(HeightForKey([]byte("884976f5")), 6, msg)
74
+
assert.Equal(HeightForKey([]byte("app.bsky.feed.post/454397e440ec")), 4, msg)
75
+
assert.Equal(HeightForKey([]byte("app.bsky.feed.post/9adeb165882c")), 8, msg)
76
+
77
+
assert.Equal(HeightForKey([]byte("R2/359107")), 2, msg)
78
+
}
+84
atproto/repo/mst/verify.go
+84
atproto/repo/mst/verify.go
···
1
+
package mst
2
+
3
+
import (
4
+
"bytes"
5
+
"fmt"
6
+
)
7
+
8
+
func (t *Tree) Verify() error {
9
+
if t.Root == nil {
10
+
return fmt.Errorf("tree missing root node")
11
+
}
12
+
return t.Root.verifyStructure(-1, nil)
13
+
}
14
+
15
+
func (n *Node) verifyStructure(height int, key []byte) error {
16
+
if n == nil {
17
+
return fmt.Errorf("nil node")
18
+
}
19
+
if n.Stub {
20
+
return fmt.Errorf("stub node")
21
+
}
22
+
if n.CID == nil && n.Dirty == false {
23
+
return fmt.Errorf("node missing CID, but not marked dirty")
24
+
}
25
+
if len(n.Entries) == 0 {
26
+
if height >= 0 {
27
+
return fmt.Errorf("empty tree node")
28
+
}
29
+
// entire tree is empty
30
+
return nil
31
+
}
32
+
33
+
if height < 0 {
34
+
// do a quick pass to compute current height
35
+
for _, e := range n.Entries {
36
+
if e.IsValue() {
37
+
height = HeightForKey(e.Key)
38
+
break
39
+
}
40
+
}
41
+
}
42
+
if height < 0 {
43
+
return fmt.Errorf("top of tree is just a pointer to child")
44
+
}
45
+
if n.Height == -1 || n.Height != height {
46
+
return fmt.Errorf("node has incorrect height: %d", n.Height)
47
+
}
48
+
49
+
lastWasChild := false
50
+
for _, e := range n.Entries {
51
+
if e.IsChild() {
52
+
if lastWasChild {
53
+
return fmt.Errorf("sibling children in entries list")
54
+
}
55
+
lastWasChild = true
56
+
if e.IsValue() {
57
+
return fmt.Errorf("entry is both a child and a value")
58
+
}
59
+
if height == 0 {
60
+
return fmt.Errorf("child below zero height")
61
+
}
62
+
if e.Child != nil {
63
+
if err := e.Child.verifyStructure(height-1, key); err != nil {
64
+
return err
65
+
}
66
+
}
67
+
} else if e.IsValue() {
68
+
lastWasChild = false
69
+
if bytes.Equal(key, e.Key) {
70
+
return fmt.Errorf("duplicate key in tree")
71
+
}
72
+
if bytes.Compare(key, e.Key) > 0 {
73
+
return fmt.Errorf("out of order keys")
74
+
}
75
+
key = e.Key
76
+
if height != HeightForKey(e.Key) {
77
+
return fmt.Errorf("wrong height for key: %d", HeightForKey(e.Key))
78
+
}
79
+
} else {
80
+
return fmt.Errorf("entry was neither child nor value")
81
+
}
82
+
}
83
+
return nil
84
+
}
+152
atproto/repo/operation.go
+152
atproto/repo/operation.go
···
1
+
package repo
2
+
3
+
import (
4
+
"fmt"
5
+
"sort"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/repo/mst"
8
+
9
+
"github.com/ipfs/go-cid"
10
+
)
11
+
12
+
type Operation struct {
13
+
Path string
14
+
Value *cid.Cid
15
+
Prev *cid.Cid
16
+
}
17
+
18
+
func (op *Operation) IsCreate() bool {
19
+
if op.Value != nil && op.Prev == nil {
20
+
return true
21
+
}
22
+
return false
23
+
}
24
+
25
+
func (op *Operation) IsUpdate() bool {
26
+
if op.Value != nil && op.Prev != nil && *op.Value != *op.Prev {
27
+
return true
28
+
}
29
+
return false
30
+
}
31
+
32
+
func (op *Operation) IsDelete() bool {
33
+
if op.Value == nil && op.Prev != nil {
34
+
return true
35
+
}
36
+
return false
37
+
}
38
+
39
+
// Mutates the tree, returning a full `Operation`
40
+
func ApplyOp(tree *mst.Tree, path string, val *cid.Cid) (*Operation, error) {
41
+
if val != nil {
42
+
prev, err := tree.Insert([]byte(path), *val)
43
+
if err != nil {
44
+
return nil, err
45
+
}
46
+
op := &Operation{
47
+
Path: path,
48
+
Value: val,
49
+
Prev: prev,
50
+
}
51
+
return op, nil
52
+
} else {
53
+
prev, err := tree.Remove([]byte(path))
54
+
if err != nil {
55
+
return nil, err
56
+
}
57
+
op := &Operation{
58
+
Path: path,
59
+
Value: val,
60
+
Prev: prev,
61
+
}
62
+
return op, nil
63
+
}
64
+
}
65
+
66
+
// Does a simple "forwards" (not inversion) check of operation
67
+
func CheckOp(tree *mst.Tree, op *Operation) error {
68
+
val, err := tree.Get([]byte(op.Path))
69
+
if err != nil {
70
+
return err
71
+
}
72
+
if op.IsCreate() || op.IsUpdate() {
73
+
if val == nil || *val != *op.Value {
74
+
return fmt.Errorf("tree value did not match op: %s %s", op.Path, val)
75
+
}
76
+
return nil
77
+
}
78
+
if op.IsDelete() {
79
+
if val != nil {
80
+
return fmt.Errorf("key still in tree after deletion op: %s", op.Path)
81
+
}
82
+
return nil
83
+
}
84
+
return fmt.Errorf("invalid operation")
85
+
}
86
+
87
+
// Applies the inversion of the `op` to the `tree`. This mutates the tree.
88
+
func InvertOp(tree *mst.Tree, op *Operation) error {
89
+
if op.IsCreate() {
90
+
prev, err := tree.Remove([]byte(op.Path))
91
+
if err != nil {
92
+
return fmt.Errorf("failed to invert op: %w", err)
93
+
}
94
+
if prev == nil || *prev != *op.Value {
95
+
return fmt.Errorf("failed to invert creation: previous record CID didn't match")
96
+
}
97
+
return nil
98
+
}
99
+
if op.IsUpdate() {
100
+
prev, err := tree.Insert([]byte(op.Path), *op.Prev)
101
+
if err != nil {
102
+
return fmt.Errorf("failed to invert op: %w", err)
103
+
}
104
+
if prev == nil || *prev != *op.Value {
105
+
return fmt.Errorf("failed to invert update: previous record CID didn't match")
106
+
}
107
+
return nil
108
+
}
109
+
if op.IsDelete() {
110
+
prev, err := tree.Insert([]byte(op.Path), *op.Prev)
111
+
if err != nil {
112
+
return fmt.Errorf("failed to invert op: %w", err)
113
+
}
114
+
if prev != nil {
115
+
return fmt.Errorf("failed to invert deletion: key was previously in tree")
116
+
}
117
+
return nil
118
+
}
119
+
return fmt.Errorf("invalid operation")
120
+
}
121
+
122
+
type opByPath []Operation
123
+
124
+
func (a opByPath) Len() int { return len(a) }
125
+
func (a opByPath) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
126
+
127
+
func (a opByPath) Less(i, j int) bool {
128
+
129
+
// sort deletions first
130
+
if a[i].IsDelete() && !a[j].IsDelete() {
131
+
return true
132
+
}
133
+
134
+
// then by path
135
+
return a[i].Path < a[j].Path
136
+
}
137
+
138
+
// re-orders operation list, and checks for duplicates
139
+
func NormalizeOps(list []Operation) ([]Operation, error) {
140
+
// TODO: can this just use the slice ref, instead of returning?
141
+
142
+
set := map[string]bool{}
143
+
for _, op := range list {
144
+
if _, ok := set[op.Path]; ok != false {
145
+
return nil, fmt.Errorf("duplicate path in operation list")
146
+
}
147
+
set[op.Path] = true
148
+
}
149
+
150
+
sort.Sort(opByPath(list))
151
+
return list, nil
152
+
}
+294
atproto/repo/operation_test.go
+294
atproto/repo/operation_test.go
···
1
+
package repo
2
+
3
+
import (
4
+
"context"
5
+
"encoding/hex"
6
+
"fmt"
7
+
"math/rand"
8
+
"testing"
9
+
10
+
"github.com/bluesky-social/indigo/atproto/repo/mst"
11
+
12
+
"github.com/ipfs/go-cid"
13
+
"github.com/ipfs/go-datastore"
14
+
blockstore "github.com/ipfs/go-ipfs-blockstore"
15
+
"github.com/multiformats/go-multihash"
16
+
"github.com/stretchr/testify/assert"
17
+
)
18
+
19
+
func randomCid() cid.Cid {
20
+
buf := make([]byte, 32)
21
+
rand.Read(buf)
22
+
c, err := cid.NewPrefixV1(cid.Raw, multihash.SHA2_256).Sum(buf)
23
+
if err != nil {
24
+
panic(err)
25
+
}
26
+
return c
27
+
}
28
+
29
+
func randomStr() string {
30
+
buf := make([]byte, 16)
31
+
rand.Read(buf)
32
+
return hex.EncodeToString(buf)
33
+
}
34
+
35
+
func debugCountEntries(n *mst.Node) int {
36
+
if n == nil {
37
+
return 0
38
+
}
39
+
count := 0
40
+
for _, e := range n.Entries {
41
+
if e.IsValue() {
42
+
count++
43
+
}
44
+
if e.IsChild() && e.Child != nil {
45
+
count += debugCountEntries(e.Child)
46
+
}
47
+
}
48
+
return count
49
+
}
50
+
51
+
func TestBasicOperation(t *testing.T) {
52
+
assert := assert.New(t)
53
+
54
+
c2, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222")
55
+
c3, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333")
56
+
et := mst.NewEmptyTree()
57
+
tree := &et
58
+
var op *Operation
59
+
var err error
60
+
61
+
op, err = ApplyOp(tree, "color/green", &c2)
62
+
assert.NoError(err)
63
+
assert.True(op.IsCreate())
64
+
assert.NoError(CheckOp(tree, op))
65
+
66
+
op, err = ApplyOp(tree, "color/brown", &c2)
67
+
assert.NoError(err)
68
+
assert.True(op.IsCreate())
69
+
assert.NoError(CheckOp(tree, op))
70
+
71
+
op, err = ApplyOp(tree, "color/brown", &c3)
72
+
assert.NoError(err)
73
+
assert.True(op.IsUpdate())
74
+
assert.Equal(c3, *op.Value)
75
+
assert.Equal(c2, *op.Prev)
76
+
assert.NoError(CheckOp(tree, op))
77
+
78
+
op, err = ApplyOp(tree, "color/brown", nil)
79
+
assert.NoError(err)
80
+
assert.True(op.IsDelete())
81
+
assert.NoError(CheckOp(tree, op))
82
+
err = InvertOp(tree, op)
83
+
assert.NoError(err)
84
+
assert.Error(CheckOp(tree, op))
85
+
86
+
op, err = ApplyOp(tree, "color/orange", &c3)
87
+
assert.NoError(err)
88
+
assert.True(op.IsCreate())
89
+
assert.NoError(CheckOp(tree, op))
90
+
err = InvertOp(tree, op)
91
+
assert.NoError(err)
92
+
assert.Error(CheckOp(tree, op))
93
+
94
+
op, err = ApplyOp(tree, "color/pink", &c3)
95
+
assert.NoError(err)
96
+
op, err = ApplyOp(tree, "color/pink", &c2)
97
+
assert.NoError(CheckOp(tree, op))
98
+
assert.True(op.IsUpdate())
99
+
err = InvertOp(tree, op)
100
+
assert.NoError(err)
101
+
assert.Error(CheckOp(tree, op))
102
+
}
103
+
104
+
func TestRandomOperations(t *testing.T) {
105
+
// single-op commits, near-empty repo
106
+
randomOperations(t, 1, 1, 50)
107
+
// single-op commits, large repo
108
+
randomOperations(t, 10000, 1, 50)
109
+
// multi-op commit
110
+
randomOperations(t, 2000, 8, 50)
111
+
}
112
+
113
+
func randomOperations(t *testing.T, size, opCount, iterations int) {
114
+
assert := assert.New(t)
115
+
ctx := context.Background()
116
+
117
+
// generate a random starting tree
118
+
startMap := make(map[string]cid.Cid, size)
119
+
for range size {
120
+
k := randomStr()
121
+
// ensure key is not already in the random set
122
+
for {
123
+
_, ok := startMap[k]
124
+
if !ok {
125
+
break
126
+
}
127
+
k = randomStr()
128
+
}
129
+
startMap[k] = randomCid()
130
+
}
131
+
mapKeys := make([]string, len(startMap))
132
+
i := 0
133
+
for k, _ := range startMap {
134
+
mapKeys[i] = k
135
+
i++
136
+
}
137
+
rand.Shuffle(len(mapKeys), func(i, j int) {
138
+
mapKeys[i], mapKeys[j] = mapKeys[j], mapKeys[i]
139
+
})
140
+
tree, err := mst.LoadTreeFromMap(startMap)
141
+
if err != nil {
142
+
t.Fatal(err)
143
+
}
144
+
assert.Equal(size, debugCountEntries(tree.Root))
145
+
assert.NoError(tree.Verify())
146
+
147
+
for range iterations {
148
+
// compute CID of the tree
149
+
startCID, err := tree.RootCID()
150
+
if err != nil {
151
+
t.Fatal(err)
152
+
}
153
+
154
+
// do some random ops
155
+
opSet := []Operation{}
156
+
var op *Operation
157
+
c := randomCid()
158
+
for range opCount {
159
+
// creations
160
+
op, err = ApplyOp(tree, randomStr(), &c)
161
+
assert.NoError(err)
162
+
opSet = append(opSet, *op)
163
+
}
164
+
165
+
for range opCount {
166
+
// deletions
167
+
op, err = ApplyOp(tree, mapKeys[rand.Intn(len(mapKeys))], nil)
168
+
assert.NoError(err)
169
+
if op.Prev != nil {
170
+
opSet = append(opSet, *op)
171
+
}
172
+
}
173
+
174
+
for range opCount {
175
+
// updates (must happen after deletions!)
176
+
k := mapKeys[rand.Intn(len(mapKeys))]
177
+
v, err := tree.Get([]byte(k))
178
+
assert.NoError(err)
179
+
if v != nil && *v != c {
180
+
op, err = ApplyOp(tree, k, &c)
181
+
assert.NoError(err)
182
+
assert.Equal(*v, *op.Prev)
183
+
assert.Equal(c, *op.Value)
184
+
if op.Prev != nil {
185
+
opSet = append(opSet, *op)
186
+
}
187
+
}
188
+
}
189
+
190
+
// extract diff as separate tree, and validate that
191
+
diffBlocks := blockstore.NewBlockstore(datastore.NewMapDatastore())
192
+
diffRoot, err := tree.WriteDiffBlocks(ctx, diffBlocks)
193
+
if err != nil {
194
+
t.Fatal(err)
195
+
}
196
+
diffTree, err := mst.LoadTreeFromStore(ctx, diffBlocks, *diffRoot)
197
+
if err != nil {
198
+
t.Fatal(err)
199
+
}
200
+
assert.NoError(tree.Verify())
201
+
202
+
// re-compute partial commit (not related to main test path)
203
+
diffCID, err := tree.RootCID()
204
+
if err != nil {
205
+
t.Fatal(err)
206
+
}
207
+
assert.Equal(*diffRoot, *diffCID)
208
+
209
+
// uncomment this to try inverting on full tree
210
+
//diffTree = tree
211
+
212
+
// check all ops against full tree (this is a redundant check)
213
+
for _, op := range opSet {
214
+
assert.NoError(CheckOp(tree, &op))
215
+
}
216
+
217
+
// sort ops (comment to disable)
218
+
opSet, err = NormalizeOps(opSet)
219
+
if err != nil {
220
+
t.Fatal(err)
221
+
}
222
+
223
+
// invert operations
224
+
for i, op := range opSet {
225
+
err := CheckOp(diffTree, &op)
226
+
fmt.Printf("loop=%d key=%s val=%s prev=%s\n", i, op.Path, op.Value, op.Prev)
227
+
assert.NoError(diffTree.Verify())
228
+
if err != nil {
229
+
//debugPrintTree(diffTree, 0)
230
+
t.Fatal(err)
231
+
}
232
+
233
+
err = InvertOp(diffTree, &op)
234
+
assert.NoError(err)
235
+
if err != nil {
236
+
t.Fatal(err)
237
+
}
238
+
}
239
+
240
+
finalCID, err := diffTree.RootCID()
241
+
if err != nil {
242
+
t.Fatal(err)
243
+
}
244
+
assert.Equal(*startCID, *finalCID)
245
+
}
246
+
247
+
// fiddle this to purge test cache
248
+
_ = 12
249
+
}
250
+
251
+
func TestNormalizeOps(t *testing.T) {
252
+
assert := assert.New(t)
253
+
254
+
c2, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu222222222")
255
+
c3, _ := cid.Decode("bafkreieqq463374bbcbeq7gpmet5rvrpeqow6t4rtjzrkhnlu333333333")
256
+
257
+
simple := []Operation{
258
+
Operation{
259
+
Path: "create-BBB",
260
+
Value: &c2,
261
+
Prev: nil,
262
+
},
263
+
Operation{
264
+
Path: "create-AAA",
265
+
Value: &c2,
266
+
Prev: nil,
267
+
},
268
+
Operation{
269
+
Path: "delete-me",
270
+
Value: nil,
271
+
Prev: &c2,
272
+
},
273
+
}
274
+
out, err := NormalizeOps(simple)
275
+
assert.NoError(err)
276
+
assert.Equal(3, len(out))
277
+
assert.Equal("delete-me", out[0].Path)
278
+
assert.Equal("create-BBB", out[2].Path)
279
+
280
+
dupes := []Operation{
281
+
Operation{
282
+
Path: "create-BBB",
283
+
Value: nil,
284
+
Prev: &c2,
285
+
},
286
+
Operation{
287
+
Path: "create-BBB",
288
+
Value: nil,
289
+
Prev: &c3,
290
+
},
291
+
}
292
+
_, err = NormalizeOps(dupes)
293
+
assert.Error(err)
294
+
}
+73
atproto/repo/repo.go
+73
atproto/repo/repo.go
···
1
+
package repo
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/repo/mst"
8
+
"github.com/bluesky-social/indigo/atproto/syntax"
9
+
10
+
"github.com/ipfs/go-cid"
11
+
"github.com/ipfs/go-datastore"
12
+
blockstore "github.com/ipfs/go-ipfs-blockstore"
13
+
)
14
+
15
+
// Version of the repo data format implemented in this package
16
+
const ATPROTO_REPO_VERSION int64 = 3
17
+
18
+
type Repo struct {
19
+
DID syntax.DID
20
+
Clock *syntax.TIDClock
21
+
Commit *Commit
22
+
23
+
RecordStore blockstore.Blockstore
24
+
MST mst.Tree
25
+
}
26
+
27
+
var ErrNotFound = errors.New("record not found in repository")
28
+
29
+
func NewRepo(did syntax.DID) Repo {
30
+
return Repo{
31
+
DID: did,
32
+
Clock: syntax.NewTIDClock(0),
33
+
Commit: nil,
34
+
RecordStore: blockstore.NewBlockstore(datastore.NewMapDatastore()),
35
+
MST: mst.NewEmptyTree(),
36
+
}
37
+
}
38
+
39
+
func (repo *Repo) GetRecordCID(ctx context.Context, collection syntax.NSID, rkey syntax.RecordKey) (*cid.Cid, error) {
40
+
path := collection.String() + "/" + rkey.String()
41
+
c, err := repo.MST.Get([]byte(path))
42
+
if err != nil {
43
+
return nil, err
44
+
}
45
+
if c == nil {
46
+
return nil, ErrNotFound
47
+
}
48
+
return c, nil
49
+
}
50
+
51
+
func (repo *Repo) GetRecordBytes(ctx context.Context, collection syntax.NSID, rkey syntax.RecordKey) ([]byte, error) {
52
+
c, err := repo.GetRecordCID(ctx, collection, rkey)
53
+
if err != nil {
54
+
return nil, err
55
+
}
56
+
blk, err := repo.RecordStore.Get(ctx, *c)
57
+
if err != nil {
58
+
return nil, err
59
+
}
60
+
// TODO: not verifying CID
61
+
return blk.RawData(), nil
62
+
}
63
+
64
+
// TODO:
65
+
// IsComplete()
66
+
// LoadFromStore
67
+
// LoadFromCAR(reader)
68
+
// WriteBlocks
69
+
// WriteCAR
70
+
// VerifyCIDs(bool)
71
+
// Export
72
+
// GetRecordStruct
73
+
// GetRecordProof
+132
atproto/repo/sync.go
+132
atproto/repo/sync.go
···
1
+
package repo
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"log/slog"
8
+
9
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
10
+
"github.com/bluesky-social/indigo/atproto/syntax"
11
+
12
+
"github.com/ipfs/go-cid"
13
+
)
14
+
15
+
// temporary/experimental method to parse and verify a firehose commit message
16
+
func VerifyCommitMessage(ctx context.Context, msg *comatproto.SyncSubscribeRepos_Commit) (*Repo, error) {
17
+
18
+
logger := slog.Default().With("did", msg.Repo, "rev", msg.Rev, "seq", msg.Seq, "time", msg.Time)
19
+
20
+
did, err := syntax.ParseDID(msg.Repo)
21
+
if err != nil {
22
+
return nil, err
23
+
}
24
+
rev, err := syntax.ParseTID(msg.Rev)
25
+
if err != nil {
26
+
return nil, err
27
+
}
28
+
_, err = syntax.ParseDatetime(msg.Time)
29
+
if err != nil {
30
+
return nil, err
31
+
}
32
+
33
+
if msg.TooBig {
34
+
logger.Warn("event with tooBig flag set")
35
+
}
36
+
if msg.Rebase {
37
+
logger.Warn("event with rebase flag set")
38
+
}
39
+
40
+
repo, err := LoadFromCAR(ctx, bytes.NewReader([]byte(msg.Blocks)))
41
+
if err != nil {
42
+
return nil, err
43
+
}
44
+
45
+
if repo.Commit.Rev != rev.String() {
46
+
return nil, fmt.Errorf("rev did not match commit")
47
+
}
48
+
if repo.Commit.DID != did.String() {
49
+
return nil, fmt.Errorf("rev did not match commit")
50
+
}
51
+
// TODO: check that commit CID matches root? re-compute?
52
+
53
+
// load out all the records
54
+
for _, op := range msg.Ops {
55
+
if (op.Action == "create" || op.Action == "update") && op.Cid != nil {
56
+
c := (*cid.Cid)(op.Cid)
57
+
nsid, rkey, err := syntax.ParseRepoPath(op.Path)
58
+
if err != nil {
59
+
return nil, fmt.Errorf("invalid repo path in ops list: %w", err)
60
+
}
61
+
val, err := repo.GetRecordCID(ctx, nsid, rkey)
62
+
if err != nil {
63
+
return nil, err
64
+
}
65
+
if *c != *val {
66
+
return nil, fmt.Errorf("record op doesn't match MST tree value")
67
+
}
68
+
_, err = repo.GetRecordBytes(ctx, nsid, rkey)
69
+
if err != nil {
70
+
return nil, err
71
+
}
72
+
}
73
+
}
74
+
75
+
// TODO: once firehose format is updated, remove this
76
+
for _, o := range msg.Ops {
77
+
if o.Action != "create" {
78
+
logger.Info("can't invert legacy op", "action", o.Action)
79
+
return repo, nil
80
+
}
81
+
}
82
+
83
+
ops, err := ParseCommitOps(msg.Ops)
84
+
if err != nil {
85
+
return nil, err
86
+
}
87
+
ops, err = NormalizeOps(ops)
88
+
if err != nil {
89
+
return nil, err
90
+
}
91
+
92
+
invTree := repo.MST.Copy()
93
+
for _, op := range ops {
94
+
if err := InvertOp(&invTree, &op); err != nil {
95
+
// print the *non-inverted* tree
96
+
//mst.DebugPrintTree(repo.MST.Root, 0)
97
+
return nil, err
98
+
}
99
+
}
100
+
// TODO: compare against previous commit for this repo?
101
+
_, err = invTree.RootCID()
102
+
103
+
logger.Info("success")
104
+
return repo, nil
105
+
}
106
+
107
+
func ParseCommitOps(ops []*comatproto.SyncSubscribeRepos_RepoOp) ([]Operation, error) {
108
+
//out := make([]mst.Operation, len(ops))
109
+
out := []Operation{}
110
+
for _, rop := range ops {
111
+
switch rop.Action {
112
+
case "create":
113
+
if rop.Cid != nil {
114
+
op := Operation{
115
+
Path: rop.Path,
116
+
Prev: nil,
117
+
Value: (*cid.Cid)(rop.Cid),
118
+
}
119
+
out = append(out, op)
120
+
} else {
121
+
return nil, fmt.Errorf("invalid repoOp: create missing CID")
122
+
}
123
+
case "delete":
124
+
return nil, fmt.Errorf("unhandled delete repoOp")
125
+
case "update":
126
+
return nil, fmt.Errorf("unhandled update repoOp")
127
+
default:
128
+
return nil, fmt.Errorf("invalid repoOp action: %s", rop.Action)
129
+
}
130
+
}
131
+
return out, nil
132
+
}
+56
atproto/repo/sync_test.go
+56
atproto/repo/sync_test.go
···
1
+
package repo
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"encoding/json"
7
+
"os"
8
+
"testing"
9
+
10
+
comatproto "github.com/bluesky-social/indigo/api/atproto"
11
+
"github.com/bluesky-social/indigo/atproto/repo/mst"
12
+
13
+
"github.com/stretchr/testify/assert"
14
+
)
15
+
16
+
func TestFirehoseTrimTopPartial(t *testing.T) {
17
+
// "failed to invert op: can not prune top of tree: MST is not complete"
18
+
testCommitFile(t, "testdata/firehose_commit_4623075231.json")
19
+
}
20
+
21
+
func TestFirehoseMergePartialNodes(t *testing.T) {
22
+
// "failed to invert op: can't merge partial nodes" (from bridgyfed PDS)
23
+
//testCommitFile(t, "testdata/firehose_commit_4621317030.json")
24
+
25
+
// "failed to invert op: can't merge partial nodes" (from bridgyfed PDS)
26
+
//testCommitFile(t, "testdata/firehose_commit_4621317332.json")
27
+
28
+
// "failed to invert op: can not merge child nodes: MST is not complete" (from bridgyfed PDS)
29
+
//testCommitFile(t, "testdata/firehose_commit_4621332152.json")
30
+
}
31
+
32
+
func testCommitFile(t *testing.T, p string) {
33
+
assert := assert.New(t)
34
+
ctx := context.Background()
35
+
36
+
body, err := os.ReadFile(p)
37
+
assert.NoError(err)
38
+
if err != nil {
39
+
t.Fail()
40
+
}
41
+
42
+
var msg comatproto.SyncSubscribeRepos_Commit
43
+
if err := json.Unmarshal(body, &msg); err != nil {
44
+
t.Fail()
45
+
}
46
+
47
+
_, err = VerifyCommitMessage(ctx, &msg)
48
+
assert.NoError(err)
49
+
if err != nil {
50
+
repo, err := LoadFromCAR(ctx, bytes.NewReader([]byte(msg.Blocks)))
51
+
if err != nil {
52
+
t.Fail()
53
+
}
54
+
mst.DebugPrintTree(repo.MST.Root, 0)
55
+
}
56
+
}
+26
atproto/repo/testdata/firehose_commit_4621317030.json
+26
atproto/repo/testdata/firehose_commit_4621317030.json
···
1
+
{
2
+
"blobs": null,
3
+
"blocks": {
4
+
"$bytes": "OqJlcm9vdHOB2CpYJQABcRIgP888UjM8r4WrcB33FNH3KFMI7iDXzM/vyvckq33gb6BndmVyc2lvbgGpBwFxEiAc+J+iVgZraHEQH5doUEVvRqSOlf+I2n+fr2GcSVSeG6JhZYikYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsZ2RlMm13dGxjcjJhcABhdNgqWCUAAXESIIEUP1lIUlLfHBVsLleTz+xmgkZglHMvQnbEBP341+GNYXbYKlglAAFxEiDKqWxSs3JmWtSR9IhwVdGFJVJWi4QjMyDZmpB/Ievu16Rha0poNzVxZG5keHMyYXAYGGF02CpYJQABcRIgB1U2OmourFJ/FdLXmmn2A4r8TEGam+ZvVlS6GlG7ZMNhdtgqWCUAAXESIK6gegsvplIyBSBFOs23wbA6xmfdDDrvrbOS+YZgegampGFrSXpkdjVjNTRzMmFwGBlhdNgqWCUAAXESIJIueCeRzc02ySC11d+m5k8kpQbqSvob8sejybMXCHgOYXbYKlglAAFxEiDu3D0atZWDfAldw7lkCmX4oaHW1dCv4XfbuzJuPUURpKRha0pzaDU3dzdsbzQyYXAYGGF02CpYJQABcRIgDPMhUJORDRvy4DM5iYwYkA8ctbcYs590EqBBUFs3vHBhdtgqWCUAAXESIH0FBYVlUBCDS7XMmHkLsIS/+uI+Vej+3L0OUJmONY6FpGFrS2hibDVyZWFpcXEyYXAXYXTYKlglAAFxEiC19nv3O98fDIPrvbAbUVMP8fgoFrjjwmapi6MtcZGLB2F22CpYJQABcRIgbHGyY+za1M1qc457pounPXfp2Dc43OkZAORC45LgY12kYWtJbXVhdHJncGcyYXAYGWF02CpYJQABcRIgQZcrUipYp7MoQ49h5GBXOfRKzCt5mfPLPIHk+4klY6VhdtgqWCUAAXESIGhu0I3PF4eNnIMUhZ3tAzoJ5FuweXT4RFF3xZufvX2dpGFrSmNycGJsaG40dTJhcBgYYXTYKlglAAFxEiCeGxEE3mRFhaL9AYYgHqWg78wU9/gPay6Z06NuvLshm2F22CpYJQABcRIgzRsw/79cg7A9bDRUecE6SWiS8w9Tr/KM1ymCjcsc0NKkYWtKaDJ5NGZydWEyMmFwGBhhdNgqWCUAAXESIFL7PABMCQLiF8/5iS4UkyfQcUt1WmULGaHNu+oCLBiZYXbYKlglAAFxEiAo0XYvxnDQ3MwjT7UIa00o8ZBwfnHoXvBUi/EVt4O8smFs2CpYJQABcRIgMRUbaaMo+bRXKVZnDh4up4Q3b/DeBuxHkltNEThLsHz8AwFxEiAGTK6c8kNeQHEZ8hSkoXQ1RmQ30EgTg4S3dxXTp860iaJhZYekYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsaGhhaGc3YTJyZjJhcABhdPZhdtgqWCUAAXESICKfP2leB3LRPMcx9vttm8+d6S8CJ5+nJP2HCNtifCZgpGFrSWQyb2M1aXlmMmFwGBlhdPZhdtgqWCUAAXESIB2+fDGtuCFUKavYxz367LzQDNbhBR+2OIopxKvZulr3pGFrSHVmbzR2dTIyYXAYGmF09mF22CpYJQABcRIgQBO0qbeJOoXgS9p+U/x4pEZTzbh1pskmb1Zx5t9s/2+kYWtJZmM2cWEyN2EyYXAYGWF09mF22CpYJQABcRIgPzeWgmN+u0moUHbzM/d5ZY381u+2eEGCxmW6Vy9W/PmkYWtJaHpuMmI2bWEyYXAYGWF09mF22CpYJQABcRIgXrcBCY7rcfcROmiPGdpineroI8zQmhnpClD66jAkmwakYWtJc2dob2JyaXIyYXAYGWF09mF22CpYJQABcRIgtJN6IIazO1P3n/wfA9cGv5LDLUllrcvQHyDhaAfzXhekYWtJdTMydDY2d2QyYXAYGWF09mF22CpYJQABcRIgIdjWvACMt+nQP4YawgRsY9SE82IF4EMO87VtGg1rUyFhbPaIAgFxEiA/zzxSMzyvhatwHfcU0fcoUwjuINfMz+/K9ySrfeBvoKZjZGlkeCBkaWQ6cGxjOjZoeTVwNnJtM3psY2RlcGhxc3NjMnA3NmNyZXZtMjIyMjIyZDduN2kyMmNzaWdYQLUY+pgU+E9glNaSASgEbTnPnR9O2dETHA6sxuq29YSuM27K6MdHEavJqT9tcx1ikxLY4BB6pQV/+I4wVfgg+GxkZGF0YdgqWCUAAXESIL8NojUibe0s+GdHFpx5GvGz3XJQQMWHioEKqpbc5OEUZHByZXbYKlglAAFxEiCUs9ALVpzzdpyQut7buZ4B7FXlXSTkdS3MR6Qtv+O8lWd2ZXJzaW9uA7sCAXESIDTMmJQyN3NA5I56qg4fiNfI3q91iQ9mZ6O5anSLRSWwomFlgqRha1giYXBwLmJza3kuZmVlZC5yZXBvc3QvM2xmeGdnN3drZmNqMmFwAGF02CpYJQABcRIg/7sexA1qhKY2vKGsJ8K3Fo0mWKOmhQxhwq6fxYRpXLdhdtgqWCUAAXESIFqJy3ug1RiBvsRYnBEi3eifF3WiTlnie8N9A8UKkDW7pGFrS2cybG1naGhtbnMyYXAXYXTYKlglAAFxEiAc+J+iVgZraHEQH5doUEVvRqSOlf+I2n+fr2GcSVSeG2F22CpYJQABcRIgCrXGp1R02Nx0+3tUKfVVu9XXTnwUqPHnKRl8D1qk7F1hbNgqWCUAAXESICZl8BPys/ta7ir0kiZ3OOyqzbRdehvHzP76cSVzL895UwFxEiBS+zwATAkC4hfP+YkuFJMn0HFLdVplCxmhzbvqAiwYmaJhZYBhbNgqWCUAAXESINqNdN12WUQ9TMLvHNTP6/xVTYUamApfu0WiR/NXL6mC0wEBcRIgvw2iNSJt7Sz4Z0cWnHka8bPdclBAxYeKgQqqltzk4RSiYWWBpGFrWCJhcHAuYnNreS5mZWVkLnJlcG9zdC8zbGZhY21zZ3ZvYmcyYXAAYXTYKlglAAFxEiA0zJiUMjdzQOSOeqoOH4jXyN6vdYkPZmejuWp0i0UlsGF22CpYJQABcRIg2DtT7U6/IEZ7A5PzoY94mr3ADUnxrpFbULVSv662bthhbNgqWCUAAXESICtBA1kz0QeiYkOUGhfF007JqZsjTvh757cWZZS/T3na+gEBcRIgooFn0kuPCY/VLzN6dXsUIBboGYdkPeYY0S5GhXaAcEKjZSR0eXBldGFwcC5ic2t5LmZlZWQucmVwb3N0Z3N1YmplY3SiY2NpZHg7YmFmeXJlaWg2NXJtdmR1cWxiM2hramw2YjRzcjYya3hxbHdrNTJ6c2M1c2g2aHJlcng3amF4eWoyZjRjdXJpeEZhdDovL2RpZDpwbGM6NWYyc29jaHZ6cGtjNnBvZXRlbTU0bjZ0L2FwcC5ic2t5LmZlZWQucG9zdC8zbGhodjVsYXF0dXIyaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowMDo0Ni4wMDBa0wEBcRIg2o103XZZRD1Mwu8c1M/r/FVNhRqYCl+7RaJH81cvqYKiYWWBpGFrWCJhcHAuYnNreS5mZWVkLnJlcG9zdC8zbGhodmRrY2J5M2UyYXAAYXTYKlglAAFxEiA93F4NaTv+XPo+6v7gIFbWlrzStja20QTE8c1FAte8GWF22CpYJQABcRIgooFn0kuPCY/VLzN6dXsUIBboGYdkPeYY0S5GhXaAcEJhbNgqWCUAAXESIAZMrpzyQ15AcRnyFKShdDVGZDfQSBODhLd3FdOnzrSJ"
5
+
},
6
+
"commit": {
7
+
"$link": "bafyreib7z46femz4v6c2w4a564knd5zikmeo4igxzth67sxxesvx3ydpua"
8
+
},
9
+
"ops": [
10
+
{
11
+
"action": "create",
12
+
"cid": {
13
+
"$link": "bafyreifcqft5es4pbgh5klztpj2xwfbac3ubtb3ehxtbrujoi2cxnadqii"
14
+
},
15
+
"path": "app.bsky.feed.repost/3lhhvdkcby3e2"
16
+
}
17
+
],
18
+
"prev": null,
19
+
"rebase": false,
20
+
"repo": "did:plc:6hy5p6rm3zlcdephqssc2p76",
21
+
"rev": "222222d7n7i22",
22
+
"seq": 4621317030,
23
+
"since": null,
24
+
"time": "2025-02-06T01:05:06.528Z",
25
+
"tooBig": false
26
+
}
+26
atproto/repo/testdata/firehose_commit_4621317332.json
+26
atproto/repo/testdata/firehose_commit_4621317332.json
···
1
+
{
2
+
"blobs": null,
3
+
"blocks": {
4
+
"$bytes": "OqJlcm9vdHOB2CpYJQABcRIgQ8oa2D2MRn6uWUgg0pK9eHDHmx8C/NT3fM26aG33qHhndmVyc2lvbgG5BwFxEiAmJ11tROjXcYIwTe1+NxaoCduQsriUcucvVhF1v19YoaJhZYikYWtYIGFwcC5ic2t5LmZlZWQubGlrZS8zbGF2Z3hhaTdrbHgyYXAAYXTYKlglAAFxEiB7nKQMYr8S2RKgG77E1ktNNDscq4NuP+w+lkZH7HNwbmF22CpYJQABcRIgPYEa4DK+G7QOmGyu0NzQzVEHWDvl3OxkpxoumQJyCzikYWtLZHNrcmRhYmozbjJhcBVhdNgqWCUAAXESIHN58X8+mERHBS2Hdcooxil07G1qgifhnc7ppkHJI0ZHYXbYKlglAAFxEiDXBvn1o/TWY6C/C2QDdxgjkPsNne639o8AsKDkgJIKGqRha1Jwb3N0LzNsNmJ4d28yMjdqbDJhcA5hdNgqWCUAAXESIIgb6WQkFP1kiJH87p3vTSbkOTZ0GxRdho8WAOnE2QMZYXbYKlglAAFxEiDPucd1mIDr+qoZpt9h31GlNlgazN24MEfTfcOoZuHstqRha0s3cmgyNHlxa3duMmFwFWF02CpYJQABcRIgVn6eOdzuM4wnxy+BGnVnHuLiOxmrhQ+jgS0OxovBrIFhdtgqWCUAAXESIDXud3kQG+U7aulDJl6bILVKOshPlnTXIV4tGyvEyuHupGFrS2NocXBqc2dkdHYyYXAVYXTYKlglAAFxEiBLlUua5lulVaXG75B/2fkXzNLHvxQI/TL3dfNca9etomF22CpYJQABcRIgaD388Uag11Z0cuLgGFYerUD+E6kSrDuZ9ck3S0hS1YukYWtLZWJueXo3am4yaTJhcBVhdNgqWCUAAXESIMOBqmuxCawNBt+QoVRZRfIBUzHF1abB9bz6q+49K8AMYXbYKlglAAFxEiAJdL4BIl+K9CcQVpCo5CttdJYBLUbpjdmtnR2mRoZFZaRha1RyZXBvc3QvM2w1dmQzbnk2Y2t4MmFwDmF02CpYJQABcRIgHz/ccWVwXohz21ERZsNK2UWNAKyYdI5mpC/XL1M+KFFhdtgqWCUAAXESINAFjz4dJIx8vO7n+afe+kF1Ap7mxxv7y+fulfD21J1spGFrS2FwcjQ1bXpwbTYyYXAXYXTYKlglAAFxEiDbVHRnfni6OyrSAv+8M3ohZgHNkOjD/LDjrmiPwxiRuWF22CpYJQABcRIgyGEh9U8hCOnL1aiNPtj7cPUSojBUywEnoduN1KvWh5lhbNgqWCUAAXESIC0XcgxVa1VxXydTYqnO7i0qKsKXXF+UgbvGJO/NdNvMiAIBcRIgQ8oa2D2MRn6uWUgg0pK9eHDHmx8C/NT3fM26aG33qHimY2RpZHggZGlkOnBsYzoyYnBib2ViNnd0Mnd0dnFwZ3ZnZ2ptbDZjcmV2bTIyMjIyMmQ3bjdxMjJjc2lnWEDLpAzW9FqZ2fYH+/Fh8hoCwiVDaif+4EofQ2Wl8wBDqQEHmHkHIsn8w5xMifRC7cmjZ1SAlMDJOcZPOkr4ThteZGRhdGHYKlglAAFxEiAmJ11tROjXcYIwTe1+NxaoCduQsriUcucvVhF1v19YoWRwcmV22CpYJQABcRIgraxiJ1FSWkO/r1saRqysNBcyyo6aiWwoZ8Gs/TX0aDNndmVyc2lvbgP6AQFxEiCbllVgVYWJO6FLGQTbX5evo7pipBZvChj2GynsCJR98KNlJHR5cGV0YXBwLmJza3kuZmVlZC5yZXBvc3Rnc3ViamVjdKJjY2lkeDtiYWZ5cmVpZWh3bzJ3aXYzMzI1dnI3czJ5bG1ldXNmZnBia2JreXRhaG5kem9mbnBpd3JpbXBleWd0aWN1cml4RmF0Oi8vZGlkOnBsYzoyYnBib2ViNnd0Mnd0dnFwZ3ZnZ2ptbDYvYXBwLmJza3kuZmVlZC5wb3N0LzNsYjVyYzRjZGJyNDJpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA0OjI3LjQ1M1qTAgFxEiCGy20I5h9gTuiSz6XQ51lPLT3fXPGtIvqUhkf48i9k+6JhZYKkYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsaGZjZGZsNzNnbDJhcABhdNgqWCUAAXESIMiGBX2KQ0qazPI1kCC7weh9GE22TviKjG4oulGUngVfYXbYKlglAAFxEiAd9kPS4SQoBR6nAB2yb+bksh5j/qdCxHP1/Gc5TBmb2aRha0podmRyNjJjemUyYXAYGGF02CpYJQABcRIg4FG08M+BIcLzxeK+6dPnpMohINBnWWThg6QAfHRdeyBhdtgqWCUAAXESIJuWVWBVhYk7oUsZBNtfl6+jumKkFm8KGPYbKewIlH3wYWz2uwIBcRIg21R0Z354ujsq0gL/vDN6IWYBzZDow/yw465oj8MYkbmiYWWCpGFrWCJhcHAuYnNreS5mZWVkLnJlcG9zdC8zbGFxNHpsZTM3ZmUyYXAAYXTYKlglAAFxEiAS8xuL6Rv0H2LJgLOxfGHRwhuY6APrWrC1v8XWcpYTYmF22CpYJQABcRIgOThGBVW5Pfd8UWem0u+ZAgl2pINbS5YZQ+kPjmOo4oykYWtLaGZjN3dkbmNjcDJhcBdhdNgqWCUAAXESIPPOGPP2H1OFkKhgwj4WZe0kvXoLlHIqRNfP2XHTcBImYXbYKlglAAFxEiADxis9NT3gkmlHv2MAsQoylgYTSDYKaHPCyIEOEzGRL2Fs2CpYJQABcRIgoxLvwh0tdq9swgX1gR4ms4OF9L6rObRAFY/nSoehe8XDAQFxEiDIhgV9ikNKmszyNZAgu8HofRhNtk74ioxuKLpRlJ4FX6JhZYKkYWtYImFwcC5ic2t5LmZlZWQucmVwb3N0LzNsaGZlNWYyc3lwcDJhcABhdPZhdtgqWCUAAXESIOkEqXYZUo6+G5V5QDnkSVqvnM8qKC9hnaJI5x0ppxrCpGFrSmhzN2xud2t1NzJhcBgYYXT2YXbYKlglAAFxEiCt6w69VSxF1+sZ26TSbxbrm+2AD910k0MgmD2P7gUEemFs9lMBcRIg884Y8/YfU4WQqGDCPhZl7SS9eguUcipE18/ZcdNwEiaiYWWAYWzYKlglAAFxEiCGy20I5h9gTuiSz6XQ51lPLT3fXPGtIvqUhkf48i9k+w"
5
+
},
6
+
"commit": {
7
+
"$link": "bafyreicdzinnqpmmiz7k4wkiedjjfplyoddzwhyc7tkpo7gnxjug355ipa"
8
+
},
9
+
"ops": [
10
+
{
11
+
"action": "create",
12
+
"cid": {
13
+
"$link": "bafyreie3szkwavmfre52csyzatnv7f5puo5gfjawn4fbr5q3fhwarfd56a"
14
+
},
15
+
"path": "app.bsky.feed.repost/3lhhvdr62cze2"
16
+
}
17
+
],
18
+
"prev": null,
19
+
"rebase": false,
20
+
"repo": "did:plc:2bpboeb6wt2wtvqpgvggjml6",
21
+
"rev": "222222d7n7q22",
22
+
"seq": 4621317332,
23
+
"since": null,
24
+
"time": "2025-02-06T01:05:06.935Z",
25
+
"tooBig": false
26
+
}
+313
atproto/repo/testdata/firehose_commit_4621332152.json
+313
atproto/repo/testdata/firehose_commit_4621332152.json
···
1
+
{
2
+
"blobs": null,
3
+
"blocks": {
4
+
"$bytes": "OqJlcm9vdHOB2CpYJQABcRIgqyrRIy8N/q+LfGxBwJ73EbEMeli2qptGcFrD2v54xjtndmVyc2lvbgHRAQFxEiB7HtprOYF6LHXoeF6r7Cr2qMG9mPb/M1TmUemZZBSuiKJhZYGkYWtYIGFwcC5ic2t5LmZlZWQubGlrZS8zbGF2dDV4cHBneTJ4YXAAYXTYKlglAAFxEiCn3KXqJzS2kKuFV5IO4oNxPhbV4nMAlasUKIVrt4mEiWF22CpYJQABcRIggVdCYKWwmOZPLVzbHVqfmeU/AQH5ZfUWAHAMpXYtUUZhbNgqWCUAAXESICU8pmcWsWl/M+oquFdFaozMCSJb/g8u9c5V0KSkZkDsUwFxEiCn3KXqJzS2kKuFV5IO4oNxPhbV4nMAlasUKIVrt4mEiaJhZYBhbNgqWCUAAXESIDxGFJEDxK5+SBLFWdlLJGYo7xCgxlJ2bKqBR+cCYd3ZUwFxEiA8RhSRA8SufkgSxVnZSyRmKO8QoMZSdmyqgUfnAmHd2aJhZYBhbNgqWCUAAXESILCRoej+qP2JXO0HvPhoqllexAudtaNmf8PMfyHWZqk9UwFxEiCwkaHo/qj9iVztB7z4aKpZXsQLnbWjZn/DzH8h1mapPaJhZYBhbNgqWCUAAXESION+97kPdHvJROaWmxVqIwy8i9bVzoLjNbVH3+pDZwX8UwFxEiDjfve5D3R7yUTmlpsVaiMMvIvW1c6C4zW1R9/qQ2cF/KJhZYBhbNgqWCUAAXESIPzbk1kmpvmyLvuSkv3Nt+LXiaO0J/qZwTiIz1Hl4YydUwFxEiD825NZJqb5si77kpL9zbfi14mjtCf6mcE4iM9R5eGMnaJhZYBhbNgqWCUAAXESIAVYW8VyIBP+U3LmMm7Nwi3v2nim45/+YZxFprw8OMekkgQBcRIgBVhbxXIgE/5TcuYybs3CLe/aeKbjn/5hnEWmvDw4x6SiYWWEpGFrWCBhcHAuYnNreS5mZWVkLmxpa2UvM2xidm8ybndrbWsyaGFwAGF02CpYJQABcRIg4fjEKsds0bi9d53csvBBiMwgemp8KkWERpotKTeeahVhdtgqWCUAAXESIAha0iqGuicbJCynZzsj0OVKN962VyNKkO+4R1e46RizpGFrS2NuNmN0aGp3ajJyYXAVYXTYKlglAAFxEiAHYChmR8wz2Vl+ijHeW9H9KxEgqZQt1zyy6f/Ee9hIEmF22CpYJQABcRIgkWu4sncXFJ2X9vLDTHk9U7wOXBNon5qhyX1iXvDBzWukYWtUcmVwb3N0LzNsYmUyZmh3c3B1MmVhcA5hdNgqWCUAAXESILe26/MVnf7GhwJlvV1q2+zaZzsdJcr9Sx9uGac5z/8XYXbYKlglAAFxEiD8uvxjctf17+bZjdwD3d0Fspbfzs08kWwYe0e49OyKwaRha0tjYWs1c3VsZGQyZmFwF2F02CpYJQABcRIgv/RdKBLzVXiAvYsRofWyR5S9nZ6NgIEmM995yrtwNFJhdtgqWCUAAXESIJSpS6Vf5eet04HvcSzGionoxZB0On601Cddl8mHiW0GYWzYKlglAAFxEiDUj9dj9ALxgq6JQvN44lL1Ls+LokbQtztEiOy57xx/B8QGAXESIL/0XSgS81V4gL2LEaH1skeUvZ2ejYCBJjPfecq7cDRSomFlh6Rha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsY2g1ZGt3Z3ZhMnNhcABhdNgqWCUAAXESIPy110h+INyVC2DAysIrvZHeI14BMR0XU1inejE3GqUQYXbYKlglAAFxEiCfu26FT7X1ss4ENufQfuaDHCyAgyH2pW0gUNE/+5i/baRha0tkZzQyanpjbXIyY2FwGBhhdNgqWCUAAXESINp9vUDilnFjGzUzroYdNf7VnUecoCaDVTEhlquwaSUgYXbYKlglAAFxEiBzCgHD5mXNan5XkNVUrjFb6oBTdqWwqv0cZnk1b7skRKRha0tmemJpeHFmY2gyaGFwGBhhdNgqWCUAAXESIHpBXTaOZzjmVcACXm0gCGc6NP6I30O1ShHqSRP0BLWlYXbYKlglAAFxEiA543K72+4fL4H2KT7OHGohg1YmAFaKjZdwfEbkdXc8eaRha0toNXg1MzVoZWcyamFwGBhhdNgqWCUAAXESIJa4zeBsWNl1/9i/NvbGfYCRLX9vpmOjJor/aytUr/nqYXbYKlglAAFxEiD138d1GcEhM8qsjP34y8/2FCO0PesBxF6oN0pkTyEQBqRha0hlbWIzeWYyamFwGBthdNgqWCUAAXESIN/NvGm4vsRayF8rJZRKTXXL47nymVBiO/Ozz4lL0k3nYXbYKlglAAFxEiByOgKXhIidsf9hL6IPoV2UcNnDtGJPmiQUhulqJKgeNKRha0l5a2FxeW1zMmphcBgaYXTYKlglAAFxEiCyh9I4eT7hq0kXEZuapWM6n+YXmOwyFs7LzsegdQWdy2F22CpYJQABcRIgq2PMxoImcjIDTM8nBba9gco8tkr5z82MF5rMPo2EEOGkYWtKaG0zeDdoM2wyM2FwGBlhdNgqWCUAAXESIBZBOA0aRagsPHXcpQ063d/HafZyfxP7JWK+B1/V8oNGYXbYKlglAAFxEiAl9bvxG1MKFf1pmZ4mTaxh0JAvIxELDHMULZXIU6rSuWFs2CpYJQABcRIgGdjmbAsOJ3L7a53UI3I6zDX3/oH38BcTXqsz9nPVF0S6AgFxEiAWQTgNGkWoLDx13KUNOt3fx2n2cn8T+yVivgdf1fKDRqJhZYKkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhobTRxdnZqejIzYXAAYXTYKlglAAFxEiDbsUEc/nr4X7ya2hvr1Y3irqiWDf8zTfakGSATgXJe7GF22CpYJQABcRIgeJ5xdgqQ+6ISpCK7SJCHQky/PSpEcPw4cLUBJVdY2hCkYWtINmZwcmJsMjNhcBgbYXTYKlglAAFxEiAtBd9ZmUKgqUdvJDf+Ltc+pBiRa+tFxYx6Q3qrBWjsfmF22CpYJQABcRIgJa3L5QSN3LCIBTzxq/ZEZNkJWtEY8Uv37u1apjmbrethbNgqWCUAAXESINrsU4cmgwU58wIZGROSQWznR+s7syr3BSyoSiIQdNJ/hAQBcRIgLQXfWZlCoKlHbyQ3/i7XPqQYkWvrRcWMekN6qwVo7H6iYWWEpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaG02cjI1ejcyM2FwAGF02CpYJQABcRIgcvYdsIaAyuupqSVCtxWFW6+4RzWhApxDatVC5aGpOmRhdtgqWCUAAXESID01OUUZ3vZyjBiaYXvSqb0r7G9AtfxrYbhEz1j5Th+dpGFrSXZkYXM3Y2MyZGFwGBphdNgqWCUAAXESICXw93sInV2H4Rh9L09SedV2viynvZ/E76tCpaQSmLT3YXbYKlglAAFxEiASrc+UR53GpkvVKyOHh0lFJ2MvtqiJo/yDIUJeQ24P5KRha0hmYTJkdnIyZGFwGBthdNgqWCUAAXESICaGiRUv7PfcnZb2syvHKrjAzvjqTLdk5B1/gbQgYODuYXbYKlglAAFxEiAvkktjKZIgPXWh1NR5k/jeWvtIkjIujh3Vr7OZv71VJaRha0VldnEyZGFwGB5hdNgqWCUAAXESIOuIulptg0IlLDvGagKw1Ygb7PfWTBmraN13e2DjVSZAYXbYKlglAAFxEiA7rSUTFHrXXjiSi/phzGJF51NRYTavgnvTg5OJwUoCsmFs2CpYJQABcRIgFrNRgXtGAx45PzUyRqeUjwQUnNu/ZGw7glhndkRFZTDrBgFxEiAl8Pd7CJ1dh+EYfS9PUnnVdr4sp72fxO+rQqWkEpi096JhZYmkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmVkZDJydzJkYXAAYXTYKlglAAFxEiAheUAq+ZxU5ERvTfYr7we4l+h+empJCpYOXneaJKgGGmF22CpYJQABcRIgtOUTgOaLffVPfIcdEtE8IjdzPSN9JALAiUbJuvAHWm6kYWtEc3EyZGFwGB9hdNgqWCUAAXESILAUiZvCqqs4a5fKxX/uXmc65wNilamSfB4kd1W8dzHwYXbYKlglAAFxEiAIeVfXH31Tj1zIFO9+gQ78NGvrrC3r1cYXdR+quGIPjaRha0N1MmRhcBggYXTYKlglAAFxEiBofsFZzD3l7+7pJ8Vht6DFRHTgRoY1YCsuZvmkE85JQWF22CpYJQABcRIg1VYYiYeOaUCCme7mj5p/zra/kihBJ3/QN7vW60WQiOykYWtIZmEyZHUzMmRhcBgbYXT2YXbYKlglAAFxEiDiTn7olF8kc4SW0aHU4/WLuF/QHVqcMxZ1mtTvJNTAJKRha0M0MmRhcBggYXT2YXbYKlglAAFxEiAYIwvZlE/7MTNcT/rDmSX93TDZWBAJ+SA3gRmd3O6xQKRha0M1MmRhcBggYXTYKlglAAFxEiCjh72IEquowBfbMlvm7n7QS7ccMo+gptVGLXuMCpEpRWF22CpYJQABcRIgDrJUNrZi59QMFQ3ziGIdQiF/u6RcFJML5f6TnOet9rakYWtDdDJkYXAYIGF02CpYJQABcRIgDnaBytrb0Cf3LALW8kCabBJaTE5WaNw80kFaeAvLVeNhdtgqWCUAAXESIHhBqCB/VdnnxDK+WiW/qKh66NASmkClYOHPnK7NvrCJpGFrRHZlMmRhcBgfYXTYKlglAAFxEiBPLYIlZh2D28+8h9zRkHc9z2dpap+JPQOzxQ3zVNd26GF22CpYJQABcRIgGAT8Lj+7FASDQH3lChR7gu8iWA2XsTGBp8P374gzKFqkYWtDcTJkYXAYIGF09mF22CpYJQABcRIguobkhEUYnGlqua7TEdUJWYmki/0nSOi6Ob3lc4R7VklhbNgqWCUAAXESIAtPwTPeRjzSnuGqyePXTMF9OTWg6J6Z8DU2UlLdZbSolwMBcRIgDnaBytrb0Cf3LALW8kCabBJaTE5WaNw80kFaeAvLVeOiYWWDpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdXYyZGFwAGF02CpYJQABcRIgMfbl8i5vYToWd47b/fUNZQ+hZD0MsI/GQRSUfLJrl7hhdtgqWCUAAXESIG/WrPmMhqGueDxV7vmstMkbMqgh/kkhDm8f4WV37kNdpGFrQ3gyZGFwGCBhdNgqWCUAAXESIFWilT9uxaUobragznpL+6o5LS7TnIUYgjsSkkHPO7WgYXbYKlglAAFxEiCOL07eR2fhQAS2CsQLetVflMMP4y2y33TZI0EImrIU0aRha0R2MzJkYXAYH2F02CpYJQABcRIg1sum48jfs2mvgLFayltsTWPIYy/1T0VcINNHlJ9SG/phdtgqWCUAAXESIM8F4y9uaZt9feX7wZofHLzu8GSF6BagiEv9zTXxLAOuYWzYKlglAAFxEiASbsjS5ZIKNZYVBu/x9pbNaxwSFmTMVJtNKlu97RyN5u4CAXESIE8tgiVmHYPbz7yH3NGQdz3PZ2lqn4k9A7PFDfNU13boomFlg6Rha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZsMmRhcABhdNgqWCUAAXESIEdv+kGtEVU6Kw/ws1AVZYzwNGxhOTIBJo2i5tuLqr4PYXbYKlglAAFxEiA9DX2X9W9NmUPLNMDYm9iqgkj2QR4zds9UESYnKeZHoqRha0NuMmRhcBggYXTYKlglAAFxEiCA7baB/wB5PaewguS8ufYN46DcGjWafRbrn9AWN0D4QGF22CpYJQABcRIgVP1y/spMhQ6mhKLdhIeaOU+vdek2ZL0WY4h0xcwT1yOkYWtDcDJkYXAYIGF09mF22CpYJQABcRIgZ5yPaeiaD2gGyXStXvJc+qo/21OAOmDxDXMy/M8S5NxhbNgqWCUAAXESILhQtI8q9tnZzfAJDvpHAGtJ5kNSWFfDZXT1JWlts46BoQMBcRIguFC0jyr22dnN8AkO+kcAa0nmQ1JYV8NldPUlaW2zjoGiYWWGpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdmYyZGFwAGF09mF22CpYJQABcRIgLGJ77hDuLN9DiugWUd1zLbn+k4vBGxzlnUYn+h8OQlikYWtDZzJkYXAYIGF09mF22CpYJQABcRIgP8ymrMC9TCLHav8w6LrbqTQF/Oy6l20lSNz2/puYMA2kYWtDaDJkYXAYIGF09mF22CpYJQABcRIg1nT+bQT6khT1qnxFcxUvrcxQI2e8Yl0G0BLqrMYx6rukYWtDaTJkYXAYIGF09mF22CpYJQABcRIgJNoIf2FFjPrbocRc80TC2J81DZ8cr6e3JnLXlz8hL/qkYWtDajJkYXAYIGF09mF22CpYJQABcRIgxY6aDMRN3Imjc9uA3vrRQnI+EwHaJ/rV1+tTvecg2likYWtDazJkYXAYIGF09mF22CpYJQABcRIgvEmmV6xSPnOqE6e4eJ0zQ+mCmr/7fQf5YJ1ToQDCSaphbPaEAQFxEiBHb/pBrRFVOisP8LNQFWWM8DRsYTkyASaNoubbi6q+D6JhZYGkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmZhMmR2bTJkYXAAYXT2YXbYKlglAAFxEiAWgI6JK8tfewevuVdhYjcZxYfx/fjAnuJLdWfPmaRT5GFs9oQBAXESIIDttoH/AHk9p7CC5Ly59g3joNwaNZp9Fuuf0BY3QPhAomFlgaRha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZvMmRhcABhdPZhdtgqWCUAAXESID7y+1vKWSWi9AOsBnQ7swH7JZMPLCGhrQ09/bnJYlagYWz21AEBcRIgJoaJFS/s99ydlvazK8cquMDO+OpMt2TkHX+BtCBg4O6iYWWBpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkd2EyZGFwAGF02CpYJQABcRIgguQtrjcVELLZ2c2BnB8HvJ20R+KSf2lObeuKIEUIJslhdtgqWCUAAXESIKE2XYySvugWUUYaiSWAkPRM3qvJXfF83tK3Z9IU9hreYWzYKlglAAFxEiAX1eg+J35ruRgaZk4/CQR+h13IBYFthoBu58jaKREm6e8CAXESIBfV6D4nfmu5GBpmTj8JBH6HXcgFgW2GgG7nyNopESbpomFlg6Rha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZ3MmRhcABhdNgqWCUAAXESIAwMiDw7jjYfmHpacVnz6By3/GcU0ciN39RgzTHsPc7eYXbYKlglAAFxEiBqBgxQA6lxJllR079EevfZdAupGBkvw0yKbK8BnDoCm6Rha0R3NDJkYXAYH2F02CpYJQABcRIgw20yDvNPQmMr0HmzJpSpDZVng973W2LduQGuFZsm2ClhdtgqWCUAAXESIMgxfLFbe1a+Q+WlezmwL4KvVpdjI5/3yp52T13HVFaapGFrQzcyZGFwGCBhdPZhdtgqWCUAAXESIGSy8mnOZ27dwvsv6vOvZ7LVyg904S6/NJG2yTNatx+QYWzYKlglAAFxEiDK8ZL5WAE8TswPAgwTsS96DN9Zcd/5nCJOIOhGeuUfMK8CAXESIMrxkvlYATxOzA8CDBOxL3oM31lx3/mcIk4g6EZ65R8womFlhKRha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZHZzMmRhcABhdPZhdtgqWCUAAXESIHEwvhH544LeUKZQJSOryGw4sddhjIU+Mfgo1ovMp/T/pGFrQ3QyZGFwGCBhdPZhdtgqWCUAAXESIGCxwohQUIsmHppPtjJaTlII/qT16v3rb7gIoLqInfSYpGFrQ3UyZGFwGCBhdPZhdtgqWCUAAXESIDYnzW2jR5ZrW0937NLiM0eKS87698UdyUZuyIHJM5B7pGFrQ3YyZGFwGCBhdPZhdtgqWCUAAXESIDg0YglLGgWkUp4WHoSpNwelw87W9awVbPzDP0QF7ZKHYWz26QIBcRIgDAyIPDuONh+YelpxWfPoHLf8ZxTRyI3f1GDNMew9zt6iYWWFpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdngyZGFwAGF09mF22CpYJQABcRIguKGknDMtdA7jJfjRwj1lW14dYNOzHryStUyYC/IaFLikYWtDeTJkYXAYIGF09mF22CpYJQABcRIgaOSuQL6XXY64K+IbjPjYBUliPTES6uAes4nE0Y6Yy1GkYWtDejJkYXAYIGF09mF22CpYJQABcRIg29AXs7My2asUhGQzaCkTcYWeA7NYD5MreNuX9zBJQKikYWtEdzIyZGFwGB9hdPZhdtgqWCUAAXESIF07WmS6/IowaZbl3evdMbPH/pefZS21a458zHg2czY8pGFrQzMyZGFwGCBhdPZhdtgqWCUAAXESIPM2xWsSwD6BlwFYA0+vOeEVA958TTnAwnVsBPx8M+bWYWz2vQEBcRIgw20yDvNPQmMr0HmzJpSpDZVng973W2LduQGuFZsm2CmiYWWCpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkdzUyZGFwAGF09mF22CpYJQABcRIgc5zuv7jNLf7Utzka2o8Ng2kbhcm7G8ZPp4XJ/5oGoyekYWtDNjJkYXAYIGF09mF22CpYJQABcRIgBCp/s3WrGcaMQRvgD2kKnnUNuv2L7Q50nMVQBdTZ9eBhbPbUAQFxEiCC5C2uNxUQstnZzYGcHwe8nbRH4pJ/aU5t64ogRQgmyaJhZYGkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmZhMmR3ZDJkYXAAYXTYKlglAAFxEiA93EfEOST8wZzYwz31xHl6TVvGW19dCr1JLXkqsxy2XGF22CpYJQABcRIglesbeYjeQmjVReI84ClHKePHeXnWul7GXyHeYwCcRzdhbNgqWCUAAXESIECS//zsXGDg1mb6k0Kp5c4/Q9ccCJt01FoXRQq5jjQLvQEBcRIgQJL//OxcYODWZvqTQqnlzj9D1xwIm3TUWhdFCrmONAuiYWWCpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJkd2IyZGFwAGF09mF22CpYJQABcRIgpwEFeRzmW5ezmwzu8JjWsj/IjinaIgS001U31YasgeikYWtDYzJkYXAYIGF09mF22CpYJQABcRIgBLyDKJtPb6i3CysBLZlbJHEii+yvF8jsv4ESXu3GPSlhbPbqAgFxEiA93EfEOST8wZzYwz31xHl6TVvGW19dCr1JLXkqsxy2XKJhZYWkYWtYI2FwcC5ic2t5LmdyYXBoLmZvbGxvdy8zbGhodmZhMmR3ZTJkYXAAYXT2YXbYKlglAAFxEiC+L1AQC/fENm9Kush3R92iPjr6qPVp/DBcuqYokJO9A6Rha0NmMmRhcBggYXT2YXbYKlglAAFxEiDtk8I5Nx+VnOXFg115DXGMVrglP7tdxLNpbScLaxyISaRha0NnMmRhcBggYXT2YXbYKlglAAFxEiBc5tJnif6mOfGo934i85fxuTeukvseqn1f0aT5YpMt46Rha0Vldm8yZGFwGB5hdPZhdtgqWCUAAXESIHN94UiMQkLnRAa4OLvm6wOVjpv/+ynxzSL+IEJ4O9JvpGFrQ3AyZGFwGCBhdPZhdtgqWCUAAXESIA7hd+4njIooRrONJyyaGdsDMvkevc/JO2g8+VncHMxnYWz21gEBcRIg64i6Wm2DQiUsO8ZqArDViBvs99ZMGato3Xd7YONVJkCiYWWBpGFrWCVhcHAuYnNreS5ncmFwaC5saXN0aXRlbS8zbGFxd3M2aHF1azIyYXAAYXTYKlglAAFxEiCzxXVQTDNZo1RUi6cm1K8rVMQ0qQX/zF69CHuq1ukZUGF22CpYJQABcRIgg7k/spSWMNWtVl4pJwn8hlo1EkFRFk3af54QDoFoqkxhbNgqWCUAAXESIK6XhQp0Z1Bsyi8B/26xVelPkp1o/SUrxbwVVL6nFH9G/AIBcRIgrpeFCnRnUGzKLwH/brFV6U+SnWj9JSvFvBVUvqcUf0aiYWWDpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJldnMyZGFwAGF09mF22CpYJQABcRIgEuEFGl293Tpd4Xpf10GCVQMlhrpmxFRv6s0JtMoggeKkYWtDdDJkYXAYIGF02CpYJQABcRIgJ8Ei1Th+RlP6/SulsCpFvY5fpEqIVRQ5SMB0rKrUi1JhdtgqWCUAAXESILXP50rvgJ0nmQUu/7vERJTqBUTjLNrlRsyWqkrjWpFapGFrUmxpc3QvM2xhcXdzNmUyYWkydWFwD2F02CpYJQABcRIgjfdOWsoWPFwM5YsYLdIYKupfaKZiYs7WSgCB36vkl/RhdtgqWCUAAXESIG1EK0I8ZT+0hg5XgFogqwDm/WEHzfeZHs8AbmPqqFXJYWzYKlglAAFxEiCNavG1x4QLL0a/TVeDmr3wMNQm6G3de5zGjR73GX9rCIQBAXESII1q8bXHhAsvRr9NV4OavfAw1Cbobd17nMaNHvcZf2sIomFlgaRha1gjYXBwLmJza3kuZ3JhcGguZm9sbG93LzNsaGh2ZmEyZXZyMmRhcABhdPZhdtgqWCUAAXESIJRfNfUHYHJ0C0a3wxu4D9OTSBoCaR+vUY8+FPMeQgUYYWz2hAEBcRIgJ8Ei1Th+RlP6/SulsCpFvY5fpEqIVRQ5SMB0rKrUi1KiYWWBpGFrWCNhcHAuYnNreS5ncmFwaC5mb2xsb3cvM2xoaHZmYTJldnUyZGFwAGF09mF22CpYJQABcRIgJeSskVxWVBuodAVXOc4fFQx7lkCNRlNj5eiYpymYCgNhbPaPAQFxEiAYBPwuP7sUBINAfeUKFHuC7yJYDZexMYGnw/fviDMoWqNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6YWMybjZvb2EzNjY0eHVid2gzdGtqdnhwaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgLGJ77hDuLN9DiugWUd1zLbn+k4vBGxzlnUYn+h8OQlijZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOm1uNHVjNjIyM2J5MjZscGk0a2V1eW1idmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESID/MpqzAvUwix2r/MOi626k0BfzsupdtJUjc9v6bmDANo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpzcGZyeG1sNWFtcW9mYzdncGpwZ3I0YWhpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDWdP5tBPqSFPWqfEVzFS+tzFAjZ7xiXQbQEuqsxjHqu6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6dG1kcXc1eGZuM3ZhemlwaHBoaGs3cGFsaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgJNoIf2FFjPrbocRc80TC2J81DZ8cr6e3JnLXlz8hL/qjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmNkbW9lbW0zc3B2eW5land0NzU0bDdyemljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIMWOmgzETdyJo3PbgN760UJyPhMB2if61dfrU73nINpYo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpleWJhcHF0bXB0Zm4zc2R0cXJ4cXVtM3FpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiC8SaZXrFI+c6oTp7h4nTND6YKav/t9B/lgnVOhAMJJqqNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6MnNtemdhaHQyNnY1aDVpczJleXhheHB3aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgPQ19l/VvTZlDyzTA2JvYqoJI9kEeM3bPVBEmJynmR6KjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmE3YWJmdmlocmNhNGZzaGxmZnp6dDZ3dGljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIBaAjokry197B6+5V2FiNxnFh/H9+MCe4kt1Z8+ZpFPko2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpyYmFsMjRiMnl2Nnp1eXJ6MjVoNnV0Z29pY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBU/XL+ykyFDqaEot2Eh5o5T6916TZkvRZjiHTFzBPXI6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6aW96djZqenF4emxtbXJocjd3bmR5eWpjaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgPvL7W8pZJaL0A6wGdDuzAfslkw8sIaGtDT39ucliVqCjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmRwZnFlZTRsNHR4NG5lNmJmdmc3M2dzMmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIGecj2nomg9oBsl0rV7yXPqqP9tTgDpg8Q1zMvzPEuTco2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzp5cTdwczZhc2NzNDdtZXJxNTJ3cDNybnNpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiC6huSERRicaWq5rtMR1QlZiaSL/SdI6Lo5veVzhHtWSaNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6eG9mYXRyeXM3NnFkdTJ6dmxhYmVyeHpiaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgL5JLYymSID11odTUeZP43lr7SJIyLo4d1a+zmb+9VSWjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOjRycGE1YzNocWNucDd3ZHhicHNud2czeWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIHEwvhH544LeUKZQJSOryGw4sddhjIU+Mfgo1ovMp/T/o2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpxNTU3aWJocW9jb2g0MmRheGFmMnNqc2JpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBgscKIUFCLJh6aT7YyWk5SCP6k9er962+4CKC6iJ30mKNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6ZG1rdDR0cGhqM2dheHVscmtla2Q0ZWx1aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgNifNbaNHlmtbT3fs0uIzR4pLzvr3xR3JRm7IgckzkHujZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmpnb2c3M2lnNHk0NzdrNm5zMml1ZGp1ZmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIDg0YglLGgWkUp4WHoSpNwelw87W9awVbPzDP0QF7ZKHo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzozamtkb3lmams1NGpiZHBwd29qemp5aHlpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBqBgxQA6lxJllR079EevfZdAupGBkvw0yKbK8BnDoCm6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6bHA3YXhobHd0bDVmbDJwcmVsbmdnbjU3aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIguKGknDMtdA7jJfjRwj1lW14dYNOzHryStUyYC/IaFLijZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOm5nbHc2YXczNWRsYWt2a240dGFmNjRnemljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIGjkrkC+l12OuCviG4z42AVJYj0xEurgHrOJxNGOmMtRo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpsaG1yeG5xZnVseTVmc254aWpwcTdleXppY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDb0BezszLZqxSEZDNoKRNxhZ4Ds1gPkyt425f3MElAqKNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6NGhudjNveGkyN2NvY2p2dWZzcGJ2NHl0aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgXTtaZLr8ijBpluXd690xs8f+l59lLbVrjnzMeDZzNjyjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOnpva25qcjV6b2pueHR4eGM3cmlneWM1eWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIPM2xWsSwD6BlwFYA0+vOeEVA958TTnAwnVsBPx8M+bWo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpzdjVnaGoydXVsaTZlc2N3a3ZtYnV1a3hpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDIMXyxW3tWvkPlpXs5sC+Cr1aXYyOf98qedk9dx1RWmqNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6NzNrNjYyaTd0Y2l1b2x6YW11bmh2NTdsaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgc5zuv7jNLf7Utzka2o8Ng2kbhcm7G8ZPp4XJ/5oGoyejZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOnZpaWluNG1hdWdtMmZ5a3pha2N5N2tjMmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIAQqf7N1qxnGjEEb4A9pCp51Dbr9i+0OdJzFUAXU2fXgo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpueWVvY2FwdHRyY2FpenJxd3FhcjZyeGlpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiBksvJpzmdu3cL7L+rzr2ey1coPdOEuvzSRtskzWrcfkKNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6Nm1tc2xhNDJwdm1temZuaXJoNTVtc3BiaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgoTZdjJK+6BZRRhqJJYCQ9Ezeq8ld8Xze0rdn0hT2Gt6jZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOnhmbGxiYjY0amZpaDJ4cnRsdG96bWQybmljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIKcBBXkc5luXs5sM7vCY1rI/yI4p2iIEtNNVN9WGrIHoo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzo0Zm9yYWZvbnhld3YyYzNrempvZnZkNmdpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiAEvIMom09vqLcLKwEtmVskcSKL7K8XyOy/gRJe7cY9KaNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6enl2cWxibGphMzdrM2w3N2prYnpqY2d5aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIglesbeYjeQmjVReI84ClHKePHeXnWul7GXyHeYwCcRzejZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmVxZmEyb2hwNDJmcGNqbDU3bTN6c3d0b2ljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIL4vUBAL98Q2b0q6yHdH3aI+Ovqo9Wn8MFy6piiQk70Do2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzp2M3dhZWZzc2tmM3ByM3VjY3o0eHc0ejRpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiDtk8I5Nx+VnOXFg115DXGMVrglP7tdxLNpbScLaxyISaNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6YmNycXh3MjJ6NzJlYXdsdmQyZDRubnY3aWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgXObSZ4n+pjnxqPd+IvOX8bk3rpL7Hqp9X9Gk+WKTLeOjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOm81Mmlpd3dhZG9lb3p2eGEyNnhhNXo2Z2ljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIHN94UiMQkLnRAa4OLvm6wOVjpv/+ynxzSL+IEJ4O9Jvo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzozaWVrZnpneTZjM2prY21pb3RubnpobGVpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiAO4XfuJ4yKKEazjScsmhnbAzL5Hr3PyTtoPPlZ3BzMZ6NlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6dXl1a2ZkM3l1dnF2NDNobXY1YnAzaWZyaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgO60lExR61144kov6YcxiRedTUWE2r4J704OTicFKArKjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOmRjcXR0dno3Y3RvaGw2aWVsYWE1aXE0NWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESIJRfNfUHYHJ0C0a3wxu4D9OTSBoCaR+vUY8+FPMeQgUYo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzpja3p2Mm12dnd1amxtamczaWc2bng2cjRpY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlqPAQFxEiAS4QUaXb3dOl3hel/XQYJVAyWGumbEVG/qzQm0yiCB4qNlJHR5cGV1YXBwLmJza3kuZ3JhcGguZm9sbG93Z3N1YmplY3R4IGRpZDpwbGM6cHR2cTNpNzNpNGdza2xpc295Yzdjb3NwaWNyZWF0ZWRBdHgYMjAyNS0wMi0wNlQwMTowNToyNi41NDJajwEBcRIgtc/nSu+AnSeZBS7/u8RElOoFROMs2uVGzJaqSuNakVqjZSR0eXBldWFwcC5ic2t5LmdyYXBoLmZvbGxvd2dzdWJqZWN0eCBkaWQ6cGxjOjZpb3lyZWdkN2FqZnlxc2s3YXN1cDZlcWljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6MDU6MjYuNTQyWo8BAXESICXkrJFcVlQbqHQFVznOHxUMe5ZAjUZTY+XomKcpmAoDo2UkdHlwZXVhcHAuYnNreS5ncmFwaC5mb2xsb3dnc3ViamVjdHggZGlkOnBsYzp4cHJobnAyeHhzczMyYjVscGNuem9xdGppY3JlYXRlZEF0eBgyMDI1LTAyLTA2VDAxOjA1OjI2LjU0MlrgAQFxEiCrKtEjLw3+r4t8bEHAnvcRsQx6WLaqm0ZwWsPa/njGO6ZjZGlkeCBkaWQ6cGxjOnZpeGl6Z2p1d2VmNW92bWl4cXhhbGZ1cGNyZXZtM2xoaHZmYW5tb3MyMmNzaWdYQLnvioSbs2Xqh0UVTxeh+6ZJ7czVQZoVoYfj24Y6iQ6ASSJ+sz3e9N/d8OY94vw4t0VnBg/aK8n2o0eSTmPmq09kZGF0YdgqWCUAAXESIHse2ms5gXosdeh4XqvsKvaowb2Y9v8zVOZR6ZlkFK6IZHByZXb2Z3ZlcnNpb24D"
5
+
},
6
+
"commit": {
7
+
"$link": "bafyreiflflisglyn72xyw7dmihaj55yrweghuwfwvknum4c2ypnp46gghm"
8
+
},
9
+
"ops": [
10
+
{
11
+
"action": "create",
12
+
"cid": {
13
+
"$link": "bafyreiayat6c4p53cqcigqd54ufbi64c54rfqdmxweyydj6d67xyqmzili"
14
+
},
15
+
"path": "app.bsky.graph.follow/3lhhvfa2dve2d"
16
+
},
17
+
{
18
+
"action": "create",
19
+
"cid": {
20
+
"$link": "bafyreibmmj564ehoftpuhcxiczi524znxh7jhc6bdmoolhkge75b6dscla"
21
+
},
22
+
"path": "app.bsky.graph.follow/3lhhvfa2dvf2d"
23
+
},
24
+
{
25
+
"action": "create",
26
+
"cid": {
27
+
"$link": "bafyreib7zstkzqf5jqrmo2x7gdulvw5jgqc7z3f2s5wsksg4637jxgbqbu"
28
+
},
29
+
"path": "app.bsky.graph.follow/3lhhvfa2dvg2d"
30
+
},
31
+
{
32
+
"action": "create",
33
+
"cid": {
34
+
"$link": "bafyreigwot7g2bh2sikplkt4ivzrkl5nzricgz54mjoqnuas5kwmmmpkxm"
35
+
},
36
+
"path": "app.bsky.graph.follow/3lhhvfa2dvh2d"
37
+
},
38
+
{
39
+
"action": "create",
40
+
"cid": {
41
+
"$link": "bafyreibe3ieh6ykfrt5nxioeltzujqwyt42q3hy4v6t3ojts26lt6ijp7i"
42
+
},
43
+
"path": "app.bsky.graph.follow/3lhhvfa2dvi2d"
44
+
},
45
+
{
46
+
"action": "create",
47
+
"cid": {
48
+
"$link": "bafyreigfr2nazrcn3se2g463qdppvukcoi7bgao2e75nlv7lko66oig2la"
49
+
},
50
+
"path": "app.bsky.graph.follow/3lhhvfa2dvj2d"
51
+
},
52
+
{
53
+
"action": "create",
54
+
"cid": {
55
+
"$link": "bafyreif4jgtfplcshzz2ue5hxb4j2m2d5gbjvp73pud7sye5koqqbqsjvi"
56
+
},
57
+
"path": "app.bsky.graph.follow/3lhhvfa2dvk2d"
58
+
},
59
+
{
60
+
"action": "create",
61
+
"cid": {
62
+
"$link": "bafyreib5bv6zp5lpjwmuhszuydmjxwfkqjepmqi6gn3m6vareytstzshui"
63
+
},
64
+
"path": "app.bsky.graph.follow/3lhhvfa2dvl2d"
65
+
},
66
+
{
67
+
"action": "create",
68
+
"cid": {
69
+
"$link": "bafyreiawqchisk6ll55qpl5zk5qwenyzywd7d7pyycpoes3vm7hztjct4q"
70
+
},
71
+
"path": "app.bsky.graph.follow/3lhhvfa2dvm2d"
72
+
},
73
+
{
74
+
"action": "create",
75
+
"cid": {
76
+
"$link": "bafyreicu7vzp5ssmquhknbfc3wcipgrzj6xxl2jwms6rmy4iotc4ye6xem"
77
+
},
78
+
"path": "app.bsky.graph.follow/3lhhvfa2dvn2d"
79
+
},
80
+
{
81
+
"action": "create",
82
+
"cid": {
83
+
"$link": "bafyreib66l5vxsszewrpia5maz2dxmyb7mszgdzmegq22dj57w44syswua"
84
+
},
85
+
"path": "app.bsky.graph.follow/3lhhvfa2dvo2d"
86
+
},
87
+
{
88
+
"action": "create",
89
+
"cid": {
90
+
"$link": "bafyreidhtshwt2e2b5uansluvvppexh2vi75wu4ahjqpcdltgl6m6exe3q"
91
+
},
92
+
"path": "app.bsky.graph.follow/3lhhvfa2dvp2d"
93
+
},
94
+
{
95
+
"action": "create",
96
+
"cid": {
97
+
"$link": "bafyreif2q3siiriytruwvono2mi5kckzrgsix7jhjduluon54vzyi62wje"
98
+
},
99
+
"path": "app.bsky.graph.follow/3lhhvfa2dvq2d"
100
+
},
101
+
{
102
+
"action": "create",
103
+
"cid": {
104
+
"$link": "bafyreibpsjfwgkmsea6xliou2r4zh6g6ll5urersf2hb3vnpwom37pkveu"
105
+
},
106
+
"path": "app.bsky.graph.follow/3lhhvfa2dvr2d"
107
+
},
108
+
{
109
+
"action": "create",
110
+
"cid": {
111
+
"$link": "bafyreidrgc7bd6pdqlpfbjsqeur2xsdmhcy5oymmqu7dd6bi22f4zj7u74"
112
+
},
113
+
"path": "app.bsky.graph.follow/3lhhvfa2dvs2d"
114
+
},
115
+
{
116
+
"action": "create",
117
+
"cid": {
118
+
"$link": "bafyreidawhbiqucqrmtb5gspwyzfutssbd7kj5pk7xvw7oaiuc5irhputa"
119
+
},
120
+
"path": "app.bsky.graph.follow/3lhhvfa2dvt2d"
121
+
},
122
+
{
123
+
"action": "create",
124
+
"cid": {
125
+
"$link": "bafyreibwe7gw3i2hszvvwt3x5tjoem2hrjf456xxyuo4srtozca4sm4qpm"
126
+
},
127
+
"path": "app.bsky.graph.follow/3lhhvfa2dvu2d"
128
+
},
129
+
{
130
+
"action": "create",
131
+
"cid": {
132
+
"$link": "bafyreibygrrassy2awsffhqwd2cksnyhuxb45vxvvqkwz7gdh5cal3msq4"
133
+
},
134
+
"path": "app.bsky.graph.follow/3lhhvfa2dvv2d"
135
+
},
136
+
{
137
+
"action": "create",
138
+
"cid": {
139
+
"$link": "bafyreidkaygfaa5joetfsuotx5chv56zoqf2sgazf7buzctmv4azyoqctm"
140
+
},
141
+
"path": "app.bsky.graph.follow/3lhhvfa2dvw2d"
142
+
},
143
+
{
144
+
"action": "create",
145
+
"cid": {
146
+
"$link": "bafyreifyugsjymznoqhogjpy2hbd2zk3lyowbu5td26jfnkmtaf7egquxa"
147
+
},
148
+
"path": "app.bsky.graph.follow/3lhhvfa2dvx2d"
149
+
},
150
+
{
151
+
"action": "create",
152
+
"cid": {
153
+
"$link": "bafyreidi4sxebpuxlwhlqk7cdogprwafjfrd2mis5lqb5m4jytiy5gglke"
154
+
},
155
+
"path": "app.bsky.graph.follow/3lhhvfa2dvy2d"
156
+
},
157
+
{
158
+
"action": "create",
159
+
"cid": {
160
+
"$link": "bafyreig32al3hmzs3gvrjbdegnucse3rqwpahm2yb6jsw6g3s73taskava"
161
+
},
162
+
"path": "app.bsky.graph.follow/3lhhvfa2dvz2d"
163
+
},
164
+
{
165
+
"action": "create",
166
+
"cid": {
167
+
"$link": "bafyreic5hnngjox4riygtfxf3xv52mnty77jph3ffw2wxdt4zr4dm4zwhq"
168
+
},
169
+
"path": "app.bsky.graph.follow/3lhhvfa2dw22d"
170
+
},
171
+
{
172
+
"action": "create",
173
+
"cid": {
174
+
"$link": "bafyreihtg3cwwewah2azoakyanh26opbcub547cnhhame5lmat6hym7g2y"
175
+
},
176
+
"path": "app.bsky.graph.follow/3lhhvfa2dw32d"
177
+
},
178
+
{
179
+
"action": "create",
180
+
"cid": {
181
+
"$link": "bafyreigigf6lcw33k27ehznfpm43al4cv5ljoyzdt734vhtwj5o4ovcwti"
182
+
},
183
+
"path": "app.bsky.graph.follow/3lhhvfa2dw42d"
184
+
},
185
+
{
186
+
"action": "create",
187
+
"cid": {
188
+
"$link": "bafyreidtttxl7ognfx7njnzzdlni6dmdnenylsn3dpde7j4fzh7zubvde4"
189
+
},
190
+
"path": "app.bsky.graph.follow/3lhhvfa2dw52d"
191
+
},
192
+
{
193
+
"action": "create",
194
+
"cid": {
195
+
"$link": "bafyreiaefj73g5nldhdiyqi34ahwscu6oug3v7ml5uhhjhgfkac5jwpv4a"
196
+
},
197
+
"path": "app.bsky.graph.follow/3lhhvfa2dw62d"
198
+
},
199
+
{
200
+
"action": "create",
201
+
"cid": {
202
+
"$link": "bafyreidewlzgttthn3o4f6zp5lz26z5s2xfa65hbf27tjenwzezvvny7sa"
203
+
},
204
+
"path": "app.bsky.graph.follow/3lhhvfa2dw72d"
205
+
},
206
+
{
207
+
"action": "create",
208
+
"cid": {
209
+
"$link": "bafyreifbgzoyzev65alfcrq2resybehujtpkxsk56f6n5uvxm7jbj5q23y"
210
+
},
211
+
"path": "app.bsky.graph.follow/3lhhvfa2dwa2d"
212
+
},
213
+
{
214
+
"action": "create",
215
+
"cid": {
216
+
"$link": "bafyreifhaecxshhglol3hgym53yjrvvsh7ei4ko2eiclju2vg7kynleb5a"
217
+
},
218
+
"path": "app.bsky.graph.follow/3lhhvfa2dwb2d"
219
+
},
220
+
{
221
+
"action": "create",
222
+
"cid": {
223
+
"$link": "bafyreiaexsbsrg2pn6uloczlaewzswzeoerix3fpc7eozp4bcjpo3rr5fe"
224
+
},
225
+
"path": "app.bsky.graph.follow/3lhhvfa2dwc2d"
226
+
},
227
+
{
228
+
"action": "create",
229
+
"cid": {
230
+
"$link": "bafyreiev5mnxtcg6ijunkrpchtqcsrzj4pdxs6owxjpmmxzb3zrqbhchg4"
231
+
},
232
+
"path": "app.bsky.graph.follow/3lhhvfa2dwd2d"
233
+
},
234
+
{
235
+
"action": "create",
236
+
"cid": {
237
+
"$link": "bafyreif6f5ibac7xyq3g6sv2zb3upxnchy5pvkhvnh6daxf2uyujbe55am"
238
+
},
239
+
"path": "app.bsky.graph.follow/3lhhvfa2dwe2d"
240
+
},
241
+
{
242
+
"action": "create",
243
+
"cid": {
244
+
"$link": "bafyreihnspbdsny7swoolrmdlv4q24mmk24ckp53lxclg2lne4fwwheije"
245
+
},
246
+
"path": "app.bsky.graph.follow/3lhhvfa2dwf2d"
247
+
},
248
+
{
249
+
"action": "create",
250
+
"cid": {
251
+
"$link": "bafyreic443jgpcp6uy47dkhxpyrphf7rxe325ex3d2vh2x6rut4wfezn4m"
252
+
},
253
+
"path": "app.bsky.graph.follow/3lhhvfa2dwg2d"
254
+
},
255
+
{
256
+
"action": "create",
257
+
"cid": {
258
+
"$link": "bafyreidtpxqurdcciltuibvyhc56n2ydswhjx773fhy42ix6ebbhqo6sn4"
259
+
},
260
+
"path": "app.bsky.graph.follow/3lhhvfa2evo2d"
261
+
},
262
+
{
263
+
"action": "create",
264
+
"cid": {
265
+
"$link": "bafyreiao4f364j4mriuenm4ne4wjugo3amzpshv5z7etw2b47fm5yhgmm4"
266
+
},
267
+
"path": "app.bsky.graph.follow/3lhhvfa2evp2d"
268
+
},
269
+
{
270
+
"action": "create",
271
+
"cid": {
272
+
"$link": "bafyreib3vusrgfd225pdreul7jq4yysf45jvcyjwv6bhxu4dsoe4csqcwi"
273
+
},
274
+
"path": "app.bsky.graph.follow/3lhhvfa2evq2d"
275
+
},
276
+
{
277
+
"action": "create",
278
+
"cid": {
279
+
"$link": "bafyreieul427kb3aoj2awrvxymn3qd6tsnebuatjd6xvddz6ctzr4qqfda"
280
+
},
281
+
"path": "app.bsky.graph.follow/3lhhvfa2evr2d"
282
+
},
283
+
{
284
+
"action": "create",
285
+
"cid": {
286
+
"$link": "bafyreias4ecruxn53u5f3yl2l7ludasvamsynotgyrkg72wnbg2muieb4i"
287
+
},
288
+
"path": "app.bsky.graph.follow/3lhhvfa2evs2d"
289
+
},
290
+
{
291
+
"action": "create",
292
+
"cid": {
293
+
"$link": "bafyreifvz7tuv34atutzsbjo7654ireu5icujyzm3lsuntewvjfogwurli"
294
+
},
295
+
"path": "app.bsky.graph.follow/3lhhvfa2evt2d"
296
+
},
297
+
{
298
+
"action": "create",
299
+
"cid": {
300
+
"$link": "bafyreibf4swjcxcwkqn2q5afk4444hyvbr5zmqenizjwhzpitctstgakam"
301
+
},
302
+
"path": "app.bsky.graph.follow/3lhhvfa2evu2d"
303
+
}
304
+
],
305
+
"prev": null,
306
+
"rebase": false,
307
+
"repo": "did:plc:vixizgjuwef5ovmixqxalfup",
308
+
"rev": "3lhhvfanmos22",
309
+
"seq": 4621332152,
310
+
"since": "3lhhvfafwmq2r",
311
+
"time": "2025-02-06T01:05:27.125Z",
312
+
"tooBig": false
313
+
}
+26
atproto/repo/testdata/firehose_commit_4623075231.json
+26
atproto/repo/testdata/firehose_commit_4623075231.json
···
1
+
{
2
+
"blobs": null,
3
+
"blocks": {
4
+
"$bytes": "OqJlcm9vdHOB2CpYJQABcRIguGgY6Gyzx0qGvu6xnMOvrseCNY9Vnayk3st3mURyKvZndmVyc2lvbgGkAQFxEiBZNQKb1PlJdfWo2pm805YjDrwKuliSKPqEOLCxNUrJlKJhZYGkYWtYG2FwcC5ic2t5LmFjdG9yLnByb2ZpbGUvc2VsZmFwAGF02CpYJQABcRIgRSEckyU/NMy/Gdrs1dSwVHYml/TnosIx/NbKvmnC21BhdtgqWCUAAXESIGT2zjtHHx/OeWyYjkAdNmJwRtOlg6ws+SBm8KaF+dKcYWz2zwEBcRIgZPbOO0cfH855bJiOQB02YnBG06WDrCz5IGbwpoX50pykZSR0eXBldmFwcC5ic2t5LmFjdG9yLnByb2ZpbGVmYXZhdGFypGNyZWbYKlglAAFVEiA2ONI9RhyxWEPzAjacznhfqSPC60Y/75u0/HVdqVgNXGRzaXplGTxUZSR0eXBlZGJsb2JobWltZVR5cGVqaW1hZ2UvanBlZ2ljcmVhdGVkQXR4GDIwMjUtMDItMDZUMDE6NDM6MjEuMjk5WmtkaXNwbGF5TmFtZWDgAQFxEiC4aBjobLPHSoa+7rGcw6+ux4I1j1WdrKTey3eZRHIq9qZjZGlkeCBkaWQ6cGxjOnN4dG9hczJzeGp2NmdoMmJpYmRnM2c0YmNyZXZtM2xoaHhpenU1d3IyZGNzaWdYQHBHEeGz9EBPjzt9aQpFHE0iOFUpCiL1SYSe5tpTO2tlHF26TAoeSWhvQnvjd/aDVfVKDnQ/dLABLB7MYnweNexkZGF0YdgqWCUAAXESIFk1ApvU+Ul19ajambzTliMOvAq6WJIo+oQ4sLE1SsmUZHByZXb2Z3ZlcnNpb24D"
5
+
},
6
+
"commit": {
7
+
"$link": "bafyreifynamoq3fty5finpxowgomhl5oy6bdld2vtwwkjxwlo6mui4rk6y"
8
+
},
9
+
"ops": [
10
+
{
11
+
"action": "create",
12
+
"cid": {
13
+
"$link": "bafyreide63hdwry7d7hhs3eyrzab2ntcobdnhjmdvqwpsidg6ctil6ostq"
14
+
},
15
+
"path": "app.bsky.actor.profile/self"
16
+
}
17
+
],
18
+
"prev": null,
19
+
"rebase": false,
20
+
"repo": "did:plc:sxtoas2sxjv6gh2bibdg3g4b",
21
+
"rev": "3lhhxizu5wr2d",
22
+
"seq": 4623075231,
23
+
"since": "3lhhxiz32k52v",
24
+
"time": "2025-02-06T01:43:22.273Z",
25
+
"tooBig": false
26
+
}
+10
gen/main.go
+10
gen/main.go
···
8
8
bsky "github.com/bluesky-social/indigo/api/bsky"
9
9
chat "github.com/bluesky-social/indigo/api/chat"
10
10
"github.com/bluesky-social/indigo/atproto/data"
11
+
atrepo "github.com/bluesky-social/indigo/atproto/repo"
12
+
atmst "github.com/bluesky-social/indigo/atproto/repo/mst"
11
13
"github.com/bluesky-social/indigo/events"
12
14
lexutil "github.com/bluesky-social/indigo/lex/util"
13
15
"github.com/bluesky-social/indigo/mst"
···
119
121
}
120
122
121
123
if err := genCfg.WriteMapEncodersToFile("atproto/data/cbor_gen.go", "data", data.GenericRecord{}, data.LegacyBlobSchema{}, data.BlobSchema{}); err != nil {
124
+
panic(err)
125
+
}
126
+
127
+
if err := genCfg.WriteMapEncodersToFile("atproto/repo/cbor_gen.go", "repo", atrepo.Commit{}); err != nil {
128
+
panic(err)
129
+
}
130
+
131
+
if err := genCfg.WriteMapEncodersToFile("atproto/repo/mst/cbor_gen.go", "mst", atmst.NodeData{}, atmst.EntryData{}); err != nil {
122
132
panic(err)
123
133
}
124
134
}