this repo has no description
functional nocache walk
authored by
whyrusleeping
and committed by
whyrusleeping
404f5ce7
95c64e97
+30
-30
+24
-22
mst/mst.go
+24
-22
mst/mst.go
···
137
137
layer int
138
138
pointer cid.Cid
139
139
validPtr bool
140
-
141
-
noCache bool
142
140
}
143
141
144
142
// NewEmptyMST reports a new empty MST using cst as its storage.
145
143
func NewEmptyMST(cst cbor.IpldStore) *MerkleSearchTree {
146
-
return createMST(cst, cid.Undef, []nodeEntry{}, 0, false)
144
+
return createMST(cst, cid.Undef, []nodeEntry{}, 0)
147
145
}
148
146
149
147
// Typescript: MST.create(storage, entries, layer, fanout) -> MST
150
-
func createMST(cst cbor.IpldStore, ptr cid.Cid, entries []nodeEntry, layer int, noCache bool) *MerkleSearchTree {
148
+
func createMST(cst cbor.IpldStore, ptr cid.Cid, entries []nodeEntry, layer int) *MerkleSearchTree {
151
149
mst := &MerkleSearchTree{
152
150
cst: cst,
153
151
pointer: ptr,
154
152
layer: layer,
155
153
entries: entries,
156
154
validPtr: ptr.Defined(),
157
-
noCache: noCache,
158
155
}
159
156
160
157
return mst
···
165
162
// This is poorly named in both implementations, because it is lazy
166
163
// Typescript: MST.load(storage, cid, layer=null, fanout) -> MST
167
164
func LoadMST(cst cbor.IpldStore, root cid.Cid) *MerkleSearchTree {
168
-
return createMST(cst, root, nil, -1, false)
169
-
}
170
-
171
-
func (mst *MerkleSearchTree) SetNoCache(v bool) {
172
-
mst.noCache = v
165
+
return createMST(cst, root, nil, -1)
173
166
}
174
167
175
168
// === "Immutability" ===
···
180
173
if entries == nil {
181
174
panic("nil entries passed to newTree")
182
175
}
183
-
return createMST(mst.cst, cid.Undef, entries, mst.layer, false)
176
+
return createMST(mst.cst, cid.Undef, entries, mst.layer)
184
177
}
185
178
186
179
// === "Getters (lazy load)" ===
···
202
195
}
203
196
// NOTE(bnewbold): Typescript version computes layer in-place here, but
204
197
// the entriesFromNodeData() helper does that for us in golang
205
-
entries, err := entriesFromNodeData(ctx, &nd, mst.cst, mst.noCache)
198
+
entries, err := entriesFromNodeData(ctx, &nd, mst.cst)
206
199
if err != nil {
207
200
return nil, err
208
201
}
209
202
if entries == nil {
210
203
panic("got nil entries from node data decoding")
211
204
}
212
-
if !mst.noCache {
213
-
mst.entries = entries
214
-
}
205
+
mst.entries = entries
215
206
return entries, nil
216
207
}
217
208
···
219
210
}
220
211
221
212
// golang-specific helper that calls in to deserializeNodeData
222
-
func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore, noCache bool) ([]nodeEntry, error) {
213
+
func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore) ([]nodeEntry, error) {
223
214
layer := -1
224
215
if len(nd.Entries) > 0 {
225
216
// NOTE(bnewbold): can compute the layer on the first KeySuffix, because for the first entry that field is a complete key
···
227
218
layer = leadingZerosOnHashBytes(firstLeaf.KeySuffix)
228
219
}
229
220
230
-
entries, err := deserializeNodeData(ctx, cst, nd, layer, noCache)
221
+
entries, err := deserializeNodeData(ctx, cst, nd, layer)
231
222
if err != nil {
232
223
return nil, err
233
224
}
···
477
468
}
478
469
479
470
checkTreeInvariant(updated)
480
-
newRoot := createMST(mst.cst, cid.Undef, updated, keyZeros, mst.noCache)
471
+
newRoot := createMST(mst.cst, cid.Undef, updated, keyZeros)
481
472
482
473
// NOTE(bnewbold): We do want to invalid the CID (because this node has
483
474
// changed, and we are "lazy" about recomputing). Setting this flag
···
907
898
return nil, err
908
899
}
909
900
910
-
return createMST(mst.cst, cid.Undef, []nodeEntry{}, layer-1, mst.noCache), nil
901
+
return createMST(mst.cst, cid.Undef, []nodeEntry{}, layer-1), nil
911
902
}
912
903
913
904
func (mst *MerkleSearchTree) createParent(ctx context.Context) (*MerkleSearchTree, error) {
···
916
907
return nil, err
917
908
}
918
909
919
-
return createMST(mst.cst, cid.Undef, []nodeEntry{mkTreeEntry(mst)}, layer+1, mst.noCache), nil
910
+
return createMST(mst.cst, cid.Undef, []nodeEntry{mkTreeEntry(mst)}, layer+1), nil
920
911
}
921
912
922
913
// === "Finding insertion points" ===
···
946
937
// key that's greater than or equal to the provided from key.
947
938
// If cb returns an error, the walk is aborted and the error is returned.
948
939
func (mst *MerkleSearchTree) WalkLeavesFrom(ctx context.Context, from string, cb func(key string, val cid.Cid) error) error {
940
+
return mst.walkLeavesFrom(ctx, from, false, cb)
941
+
}
942
+
943
+
func (mst *MerkleSearchTree) WalkLeavesFromNocache(ctx context.Context, from string, cb func(key string, val cid.Cid) error) error {
944
+
return mst.walkLeavesFrom(ctx, from, true, cb)
945
+
}
946
+
947
+
func (mst *MerkleSearchTree) walkLeavesFrom(ctx context.Context, from string, nocache bool, cb func(key string, val cid.Cid) error) error {
949
948
index, err := mst.findGtOrEqualLeafIndex(ctx, from)
950
949
if err != nil {
951
950
return err
···
959
958
if index > 0 {
960
959
prev := entries[index-1]
961
960
if !prev.isUndefined() && prev.isTree() {
962
-
if err := prev.Tree.WalkLeavesFrom(ctx, from, cb); err != nil {
961
+
if err := prev.Tree.walkLeavesFrom(ctx, from, nocache, cb); err != nil {
963
962
return fmt.Errorf("walk leaves %d: %w", index, err)
964
963
}
965
964
}
···
971
970
return err
972
971
}
973
972
} else {
974
-
if err := e.Tree.WalkLeavesFrom(ctx, from, cb); err != nil {
973
+
if err := e.Tree.walkLeavesFrom(ctx, from, nocache, cb); err != nil {
975
974
return fmt.Errorf("walk leaves from (%d): %w", i, err)
975
+
}
976
+
if nocache {
977
+
e.Tree = nil
976
978
}
977
979
}
978
980
}
+2
-2
mst/mst_test.go
+2
-2
mst/mst_test.go
···
36
36
37
37
ctx := context.Background()
38
38
cst := util.CborStore(blockstore.NewBlockstore(datastore.NewMapDatastore()))
39
-
mst := createMST(cst, cid.Undef, []nodeEntry{}, -1, false)
39
+
mst := createMST(cst, cid.Undef, []nodeEntry{}, -1)
40
40
41
41
// NOTE: these were previously generated randomly, but the random seed behavior changed
42
42
vals := map[string]cid.Cid{
···
364
364
365
365
func cidMapToMst(t testing.TB, bs blockstore.Blockstore, m map[string]cid.Cid) *MerkleSearchTree {
366
366
cst := util.CborStore(bs)
367
-
mt := createMST(cst, cid.Undef, []nodeEntry{}, -1, false)
367
+
mt := createMST(cst, cid.Undef, []nodeEntry{}, -1)
368
368
369
369
for k, v := range m {
370
370
nmst, err := mt.Add(context.TODO(), k, v, -1)
+3
-3
mst/mst_util.go
+3
-3
mst/mst_util.go
···
66
66
}
67
67
68
68
// Typescript: deserializeNodeData(storage, data, layer)
69
-
func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int, noCache bool) ([]nodeEntry, error) {
69
+
func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int) ([]nodeEntry, error) {
70
70
entries := []nodeEntry{}
71
71
if nd.Left != nil {
72
72
// Note: like Typescript, this is actually a lazy load
73
73
entries = append(entries, nodeEntry{
74
74
Kind: entryTree,
75
-
Tree: createMST(cst, *nd.Left, nil, layer-1, noCache),
75
+
Tree: createMST(cst, *nd.Left, nil, layer-1),
76
76
})
77
77
}
78
78
···
100
100
if e.Tree != nil {
101
101
entries = append(entries, nodeEntry{
102
102
Kind: entryTree,
103
-
Tree: createMST(cst, *e.Tree, nil, layer-1, noCache),
103
+
Tree: createMST(cst, *e.Tree, nil, layer-1),
104
104
Key: keyStr,
105
105
})
106
106
}
+1
-3
repo/stream.go
+1
-3
repo/stream.go
···
137
137
138
138
t := mst.LoadMST(cst, sc.Data)
139
139
140
-
t.SetNoCache(true)
141
-
142
-
if err := t.WalkLeavesFrom(ctx, prefix, func(k string, val cid.Cid) error {
140
+
if err := t.WalkLeavesFromNocache(ctx, prefix, func(k string, val cid.Cid) error {
143
141
if err := bs.View(val, func(data []byte) error {
144
142
return cb(k, val, data)
145
143
}); err != nil {