this repo has no description
inductive firehose interop tests (#963)
Pulls in the JSON-formatted MST inversion tests from @dholms's
typescript implementation.
Updates the `indigo:atproto/repo/mst` code to include additional "proof"
blocks for "create" record ops, not just "delete" record ops. I'm not
sure the book is totally closed on that decision, but going to get this
implemented and merged for now, so that implementation/interop (between
typescript and golang) is not a barrier or issue in the protocol
decision.
+280
-38
+143
atproto/repo/inductive_interop_test.go
+143
atproto/repo/inductive_interop_test.go
···
1
+
package repo
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"os"
7
+
"testing"
8
+
9
+
"github.com/bluesky-social/indigo/atproto/repo/mst"
10
+
11
+
"github.com/ipfs/go-cid"
12
+
"github.com/ipfs/go-datastore"
13
+
blockstore "github.com/ipfs/go-ipfs-blockstore"
14
+
"github.com/stretchr/testify/assert"
15
+
)
16
+
17
+
type CommitProofFixture struct {
18
+
Comment string `json:"comment"`
19
+
LeafValue string `json:"leafValue"`
20
+
Keys []string `json:"keys"`
21
+
Additions []string `json:"adds"`
22
+
Deletions []string `json:"dels"`
23
+
RootBeforeCommit string `json:"rootBeforeCommit"`
24
+
RootAfterCommit string `json:"rootAfterCommit"`
25
+
BlocksInProof []string `json:"blocksInProof"`
26
+
}
27
+
28
+
func LoadCommitProofFixtures(p string) []CommitProofFixture {
29
+
b, err := os.ReadFile(p)
30
+
if err != nil {
31
+
panic(err)
32
+
}
33
+
var fixtures []CommitProofFixture
34
+
if err := json.Unmarshal(b, &fixtures); err != nil {
35
+
panic(err)
36
+
}
37
+
return fixtures
38
+
}
39
+
40
+
func (f *CommitProofFixture) RecordCID() cid.Cid {
41
+
c, err := cid.Decode(f.LeafValue)
42
+
if err != nil {
43
+
panic(err)
44
+
}
45
+
return c
46
+
}
47
+
48
+
func (f *CommitProofFixture) Tree() mst.Tree {
49
+
m := map[string]cid.Cid{}
50
+
c := f.RecordCID()
51
+
for _, k := range f.Keys {
52
+
m[k] = c
53
+
}
54
+
tree, err := mst.LoadTreeFromMap(m)
55
+
if err != nil {
56
+
panic(err)
57
+
}
58
+
return *tree
59
+
}
60
+
61
+
func (f *CommitProofFixture) Operations() []Operation {
62
+
c := f.RecordCID()
63
+
ops := []Operation{}
64
+
for _, key := range f.Additions {
65
+
ops = append(ops, Operation{
66
+
Path: key,
67
+
Value: &c,
68
+
Prev: nil,
69
+
})
70
+
}
71
+
for _, key := range f.Deletions {
72
+
ops = append(ops, Operation{
73
+
Path: key,
74
+
Value: nil,
75
+
Prev: &c,
76
+
})
77
+
}
78
+
return ops
79
+
}
80
+
81
+
func (f *CommitProofFixture) Test(t *testing.T) {
82
+
assert := assert.New(t)
83
+
ctx := context.Background()
84
+
85
+
tree := f.Tree()
86
+
ops := f.Operations()
87
+
88
+
// verify "before" tree CID
89
+
before, err := tree.RootCID()
90
+
if err != nil {
91
+
t.Fatal(err)
92
+
}
93
+
assert.Equal(f.RootBeforeCommit, before.String())
94
+
assert.NoError(tree.Verify())
95
+
96
+
// apply all ops, and generate diff
97
+
for _, o := range ops {
98
+
_, err := ApplyOp(&tree, o.Path, o.Value)
99
+
if err != nil {
100
+
t.Fatal(err)
101
+
}
102
+
}
103
+
diffBlocks := blockstore.NewBlockstore(datastore.NewMapDatastore())
104
+
diffRoot, err := tree.WriteDiffBlocks(ctx, diffBlocks)
105
+
if err != nil {
106
+
t.Fatal(err)
107
+
}
108
+
assert.Equal(f.RootAfterCommit, diffRoot.String())
109
+
diffTree, err := mst.LoadTreeFromStore(ctx, diffBlocks, *diffRoot)
110
+
if err != nil {
111
+
t.Fatal(err)
112
+
}
113
+
assert.NoError(diffTree.Verify())
114
+
115
+
// verify inclusion of blocks in diff
116
+
for _, cstr := range f.BlocksInProof {
117
+
c, err := cid.Decode(cstr)
118
+
if err != nil {
119
+
t.Fatal(err)
120
+
}
121
+
_, err = diffBlocks.Get(ctx, c)
122
+
assert.NoError(err)
123
+
}
124
+
125
+
// invert all ops
126
+
for _, o := range ops {
127
+
assert.NoError(CheckOp(diffTree, &o))
128
+
assert.NoError(InvertOp(diffTree, &o))
129
+
}
130
+
131
+
// verify "before" tree CID
132
+
checkCID, err := diffTree.RootCID()
133
+
assert.NoError(err)
134
+
assert.Equal(f.RootBeforeCommit, checkCID.String())
135
+
}
136
+
137
+
func TestCommitProofFixtures(t *testing.T) {
138
+
fixtures := LoadCommitProofFixtures("testdata/commit-proof-fixtures.json")
139
+
140
+
for _, f := range fixtures {
141
+
f.Test(t)
142
+
}
143
+
}
+37
atproto/repo/mst/node.go
+37
atproto/repo/mst/node.go
···
233
233
return 0, nil
234
234
}
235
235
236
+
// helper to mark nodes as "dirty" if they are needed to "prove" something about the key. used to generate invertable operation diffs.
237
+
func proveMutation(n *Node, key []byte) error {
238
+
for i, e := range n.Entries {
239
+
if e.IsValue() {
240
+
if bytes.Compare(key, e.Key) < 0 {
241
+
return nil
242
+
}
243
+
}
244
+
if e.IsChild() {
245
+
// first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child
246
+
if i+1 < len(n.Entries) {
247
+
next := n.Entries[i+1]
248
+
if next.IsValue() && bytes.Compare(key, next.Key) > 0 {
249
+
continue
250
+
}
251
+
}
252
+
if e.Child == nil {
253
+
return fmt.Errorf("can't prove mutation: %w", ErrPartialTree)
254
+
}
255
+
order, err := e.Child.compareKey(key, true)
256
+
if err != nil {
257
+
return err
258
+
}
259
+
if order > 0 {
260
+
// key comes after this entire child sub-tree
261
+
continue
262
+
}
263
+
if order < 0 {
264
+
return nil
265
+
}
266
+
// key falls inside this child sub-tree
267
+
return proveMutation(e.Child, key)
268
+
}
269
+
}
270
+
return nil
271
+
}
272
+
236
273
// helper function, mostly for testing or development, which redusively inserts key/CID pairs into a `map[string]cid.Cid
237
274
func (n *Node) writeToMap(m map[string]cid.Cid) error {
238
275
if m == nil {
+3
atproto/repo/mst/node_insert.go
+3
atproto/repo/mst/node_insert.go
+1
-38
atproto/repo/mst/node_remove.go
+1
-38
atproto/repo/mst/node_remove.go
···
1
1
package mst
2
2
3
3
import (
4
-
"bytes"
5
4
"fmt"
6
5
"slices"
7
6
···
62
61
}
63
62
64
63
// marks adjacent child nodes dirty to include as "proof"
65
-
proveDeletion(n, key)
64
+
proveMutation(n, key)
66
65
67
66
// check if top of node is now just a pointer
68
67
if top {
···
87
86
}
88
87
}
89
88
return n, prev, nil
90
-
}
91
-
92
-
func proveDeletion(n *Node, key []byte) error {
93
-
for i, e := range n.Entries {
94
-
if e.IsValue() {
95
-
if bytes.Compare(key, e.Key) < 0 {
96
-
return nil
97
-
}
98
-
}
99
-
if e.IsChild() {
100
-
// first, see if there is a next entry as a value which this key would be after; if so we can skip checking this child
101
-
if i+1 < len(n.Entries) {
102
-
next := n.Entries[i+1]
103
-
if next.IsValue() && bytes.Compare(key, next.Key) > 0 {
104
-
continue
105
-
}
106
-
}
107
-
if e.Child == nil {
108
-
return fmt.Errorf("can't prove deletion: %w", ErrPartialTree)
109
-
}
110
-
order, err := e.Child.compareKey(key, true)
111
-
if err != nil {
112
-
return err
113
-
}
114
-
if order > 0 {
115
-
// key comes after this entire child sub-tree
116
-
continue
117
-
}
118
-
if order < 0 {
119
-
return nil
120
-
}
121
-
// key falls inside this child sub-tree
122
-
return proveDeletion(e.Child, key)
123
-
}
124
-
}
125
-
return nil
126
89
}
127
90
128
91
func mergeNodes(left *Node, right *Node) (*Node, error) {
+96
atproto/repo/testdata/commit-proof-fixtures.json
+96
atproto/repo/testdata/commit-proof-fixtures.json
···
1
+
[
2
+
{
3
+
"comment": "two deep split",
4
+
"leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
5
+
"keys": [
6
+
"A0/501344",
7
+
"B1/293486",
8
+
"C0/535043",
9
+
"E0/922708",
10
+
"F1/415452",
11
+
"G0/714257"
12
+
],
13
+
"adds": ["D2/915466"],
14
+
"dels": [],
15
+
"rootBeforeCommit": "bafyreibthlzzn3rwvmomwf4dz6utt7yeh5eyn6qwbumvjfv35gwanh7ovq",
16
+
"rootAfterCommit": "bafyreidb6bxxylhmlzs4a6ruhcunv3fd32o6i5phlzkmjk6arletj2ua6a",
17
+
"blocksInProof": [
18
+
"bafyreidb6bxxylhmlzs4a6ruhcunv3fd32o6i5phlzkmjk6arletj2ua6a",
19
+
"bafyreifjsxnultnc3tbvnrawqpmrk6d76ymcstwcr5e3hn6u472nasb2xq",
20
+
"bafyreibzch5k5j5xkg6dcwmur2p6jqwavyjhdtvifr6g2gnccwhixibzsi",
21
+
"bafyreiamcu5ud3j4ovclrgq2sdyev5oajsmpnl2fdu5ffgpfint64n2jme",
22
+
"bafyreidxvw3sbdg4t5b2mbtozitnyu7kjien2zcrtgdj4ssgmjb72mzawe"
23
+
]
24
+
},
25
+
{
26
+
"comment": "two deep leafless split",
27
+
"leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
28
+
"keys": ["A0/501344", "B0/436099", "D0/360671", "E0/922708"],
29
+
"adds": ["C2/953910"],
30
+
"dels": [],
31
+
"rootBeforeCommit": "bafyreid7jnvjg7mr4akmyf7rtaz47duex2l47rz36nvs4i7yjnpuhfmehe",
32
+
"rootAfterCommit": "bafyreih2ry5gae5r4m47unhhuw4w2qjdhe6oprw3w2uico2tzbflwi74eu",
33
+
"blocksInProof": [
34
+
"bafyreih2ry5gae5r4m47unhhuw4w2qjdhe6oprw3w2uico2tzbflwi74eu",
35
+
"bafyreiag5ata5gtynbpef26l4kus2uz4nshuo526h275oljwlm5dwsvhqm",
36
+
"bafyreiaybgpm7ahyiy7fko7c4czjokhzajvimot6lfi6mxqzw2bzwoddn4",
37
+
"bafyreiheqxxydll4b4zlyemmegb7q3chs7aacczuotpxkqils6bufnsyse",
38
+
"bafyreigkijiuasyl5x4f2j3kxzou2vsdyc3vockx63r6bvgoip4ybhj2sa"
39
+
]
40
+
},
41
+
{
42
+
"comment": "add on edge with neighbor two layers down",
43
+
"leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
44
+
"keys": ["A0/501344", "B2/303249", "C0/535043"],
45
+
"adds": ["D2/915466"],
46
+
"dels": [],
47
+
"rootBeforeCommit": "bafyreifoy7ierkqljk37wozudqhqjuuahjnubqvd3qprx5ocwcfrx5v3hm",
48
+
"rootAfterCommit": "bafyreid2i3nxmsvv3ifb53nlkjh3qaymygrrxuno6z22gctzdme5lbptky",
49
+
"blocksInProof": [
50
+
"bafyreid2i3nxmsvv3ifb53nlkjh3qaymygrrxuno6z22gctzdme5lbptky",
51
+
"bafyreiagiwrefvm27hvgryirykp7reqcpz56v6txzksgbargjlibtpsqwu",
52
+
"bafyreiewdvzcopoza6bdntvhmvdfqeolql6sckkiu75jpvfnwwnfi57jia"
53
+
]
54
+
},
55
+
{
56
+
"comment": "merge and split in multi-op commit",
57
+
"leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
58
+
"keys": ["A0/501344", "B2/303249", "D2/915466", "E0/922708"],
59
+
"adds": ["C2/953910"],
60
+
"dels": ["B2/303249", "D2/915466"],
61
+
"rootBeforeCommit": "bafyreielnllkafudlseizljjx32rkkivlgxziqayhctgbxncw2srrox7ny",
62
+
"rootAfterCommit": "bafyreih6464tr7ue67qgllhiekgfmwiz45zuthrv72gwi2tjpuu5dbxt3a",
63
+
"blocksInProof": [
64
+
"bafyreih6464tr7ue67qgllhiekgfmwiz45zuthrv72gwi2tjpuu5dbxt3a",
65
+
"bafyreihexby6fnhajsjzzqkmegqpqt2lrr3rpesyl6kt3t3xppid7tuvfy",
66
+
"bafyreiciix65xuk62hu6ew6jdy3m2swqstvnuhuwcwffidk3nduf7eaoh4",
67
+
"bafyreieneexkszoung4zc5jzkjukjbbxm74ukz6mylydj7q2v42zqp6vmy",
68
+
"bafyreidxvw3sbdg4t5b2mbtozitnyu7kjien2zcrtgdj4ssgmjb72mzawe"
69
+
]
70
+
},
71
+
{
72
+
"comment": "complex multi-op commit",
73
+
"leafValue": "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454",
74
+
"keys": [
75
+
"B0/436099",
76
+
"C2/953910",
77
+
"D0/360671",
78
+
"E2/413113",
79
+
"F0/606463",
80
+
"H0/740256"
81
+
],
82
+
"adds": ["A2/239654", "G2/536869"],
83
+
"dels": ["C2/953910"],
84
+
"rootBeforeCommit": "bafyreiej4jqggfhidabjfrjgogdwed5eglhnboepxscbwfrss4uclnrrmi",
85
+
"rootAfterCommit": "bafyreifykpu67c4w4ynkx4lvjfjwxdofax6gx7j2wxrl6ewt3yslezcb6i",
86
+
"blocksInProof": [
87
+
"bafyreifykpu67c4w4ynkx4lvjfjwxdofax6gx7j2wxrl6ewt3yslezcb6i",
88
+
"bafyreig5pe2hdnhfbqleo6yyipkw3tdiju7tlm4sqp7btsiicxe4tex5de",
89
+
"bafyreievjgro75jk6ma3xwuqvalsydtzgvbbaduhazbvajvslaf3l6kcxu",
90
+
"bafyreieax6243224jnbout6ynursux2dvt6fabonofdu47dxupkxvmflvu",
91
+
"bafyreie44qmlnwlyeh6ubb2eocfko6st7gmbarplmcci6c7ilx24vh4iym",
92
+
"bafyreihlhqn4quwcgbum5g4wzkini2c42j7zi5dsjdgkzm55jxyvebndue",
93
+
"bafyreiggcbzkb2wgenvyfhkh2nggf7pohb7uzjm6bs7hixhjxw2xpmnq6u"
94
+
]
95
+
}
96
+
]