this repo has no description
flag in mst code to avoid caching internal stuff, makes oncethrough iteration nicer
authored by
whyrusleeping
and committed by
whyrusleeping
95c64e97
a5f57b26
+30
-25
+20
-11
mst/mst.go
+20
-11
mst/mst.go
···
137
137
layer int
138
138
pointer cid.Cid
139
139
validPtr bool
140
+
141
+
noCache bool
140
142
}
141
143
142
144
// NewEmptyMST reports a new empty MST using cst as its storage.
143
145
func NewEmptyMST(cst cbor.IpldStore) *MerkleSearchTree {
144
-
return createMST(cst, cid.Undef, []nodeEntry{}, 0)
146
+
return createMST(cst, cid.Undef, []nodeEntry{}, 0, false)
145
147
}
146
148
147
149
// Typescript: MST.create(storage, entries, layer, fanout) -> MST
148
-
func createMST(cst cbor.IpldStore, ptr cid.Cid, entries []nodeEntry, layer int) *MerkleSearchTree {
150
+
func createMST(cst cbor.IpldStore, ptr cid.Cid, entries []nodeEntry, layer int, noCache bool) *MerkleSearchTree {
149
151
mst := &MerkleSearchTree{
150
152
cst: cst,
151
153
pointer: ptr,
152
154
layer: layer,
153
155
entries: entries,
154
156
validPtr: ptr.Defined(),
157
+
noCache: noCache,
155
158
}
156
159
157
160
return mst
···
162
165
// This is poorly named in both implementations, because it is lazy
163
166
// Typescript: MST.load(storage, cid, layer=null, fanout) -> MST
164
167
func LoadMST(cst cbor.IpldStore, root cid.Cid) *MerkleSearchTree {
165
-
return createMST(cst, root, nil, -1)
168
+
return createMST(cst, root, nil, -1, false)
169
+
}
170
+
171
+
func (mst *MerkleSearchTree) SetNoCache(v bool) {
172
+
mst.noCache = v
166
173
}
167
174
168
175
// === "Immutability" ===
···
173
180
if entries == nil {
174
181
panic("nil entries passed to newTree")
175
182
}
176
-
return createMST(mst.cst, cid.Undef, entries, mst.layer)
183
+
return createMST(mst.cst, cid.Undef, entries, mst.layer, false)
177
184
}
178
185
179
186
// === "Getters (lazy load)" ===
···
195
202
}
196
203
// NOTE(bnewbold): Typescript version computes layer in-place here, but
197
204
// the entriesFromNodeData() helper does that for us in golang
198
-
entries, err := entriesFromNodeData(ctx, &nd, mst.cst)
205
+
entries, err := entriesFromNodeData(ctx, &nd, mst.cst, mst.noCache)
199
206
if err != nil {
200
207
return nil, err
201
208
}
202
209
if entries == nil {
203
210
panic("got nil entries from node data decoding")
204
211
}
205
-
mst.entries = entries
212
+
if !mst.noCache {
213
+
mst.entries = entries
214
+
}
206
215
return entries, nil
207
216
}
208
217
···
210
219
}
211
220
212
221
// golang-specific helper that calls in to deserializeNodeData
213
-
func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore) ([]nodeEntry, error) {
222
+
func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore, noCache bool) ([]nodeEntry, error) {
214
223
layer := -1
215
224
if len(nd.Entries) > 0 {
216
225
// NOTE(bnewbold): can compute the layer on the first KeySuffix, because for the first entry that field is a complete key
···
218
227
layer = leadingZerosOnHashBytes(firstLeaf.KeySuffix)
219
228
}
220
229
221
-
entries, err := deserializeNodeData(ctx, cst, nd, layer)
230
+
entries, err := deserializeNodeData(ctx, cst, nd, layer, noCache)
222
231
if err != nil {
223
232
return nil, err
224
233
}
···
468
477
}
469
478
470
479
checkTreeInvariant(updated)
471
-
newRoot := createMST(mst.cst, cid.Undef, updated, keyZeros)
480
+
newRoot := createMST(mst.cst, cid.Undef, updated, keyZeros, mst.noCache)
472
481
473
482
// NOTE(bnewbold): We do want to invalid the CID (because this node has
474
483
// changed, and we are "lazy" about recomputing). Setting this flag
···
898
907
return nil, err
899
908
}
900
909
901
-
return createMST(mst.cst, cid.Undef, []nodeEntry{}, layer-1), nil
910
+
return createMST(mst.cst, cid.Undef, []nodeEntry{}, layer-1, mst.noCache), nil
902
911
}
903
912
904
913
func (mst *MerkleSearchTree) createParent(ctx context.Context) (*MerkleSearchTree, error) {
···
907
916
return nil, err
908
917
}
909
918
910
-
return createMST(mst.cst, cid.Undef, []nodeEntry{mkTreeEntry(mst)}, layer+1), nil
919
+
return createMST(mst.cst, cid.Undef, []nodeEntry{mkTreeEntry(mst)}, layer+1, mst.noCache), nil
911
920
}
912
921
913
922
// === "Finding insertion points" ===
+2
-2
mst/mst_test.go
+2
-2
mst/mst_test.go
···
36
36
37
37
ctx := context.Background()
38
38
cst := util.CborStore(blockstore.NewBlockstore(datastore.NewMapDatastore()))
39
-
mst := createMST(cst, cid.Undef, []nodeEntry{}, -1)
39
+
mst := createMST(cst, cid.Undef, []nodeEntry{}, -1, false)
40
40
41
41
// NOTE: these were previously generated randomly, but the random seed behavior changed
42
42
vals := map[string]cid.Cid{
···
364
364
365
365
func cidMapToMst(t testing.TB, bs blockstore.Blockstore, m map[string]cid.Cid) *MerkleSearchTree {
366
366
cst := util.CborStore(bs)
367
-
mt := createMST(cst, cid.Undef, []nodeEntry{}, -1)
367
+
mt := createMST(cst, cid.Undef, []nodeEntry{}, -1, false)
368
368
369
369
for k, v := range m {
370
370
nmst, err := mt.Add(context.TODO(), k, v, -1)
+3
-3
mst/mst_util.go
+3
-3
mst/mst_util.go
···
66
66
}
67
67
68
68
// Typescript: deserializeNodeData(storage, data, layer)
69
-
func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int) ([]nodeEntry, error) {
69
+
func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int, noCache bool) ([]nodeEntry, error) {
70
70
entries := []nodeEntry{}
71
71
if nd.Left != nil {
72
72
// Note: like Typescript, this is actually a lazy load
73
73
entries = append(entries, nodeEntry{
74
74
Kind: entryTree,
75
-
Tree: createMST(cst, *nd.Left, nil, layer-1),
75
+
Tree: createMST(cst, *nd.Left, nil, layer-1, noCache),
76
76
})
77
77
}
78
78
···
100
100
if e.Tree != nil {
101
101
entries = append(entries, nodeEntry{
102
102
Kind: entryTree,
103
-
Tree: createMST(cst, *e.Tree, nil, layer-1),
103
+
Tree: createMST(cst, *e.Tree, nil, layer-1, noCache),
104
104
Key: keyStr,
105
105
})
106
106
}
+2
-2
repo/repo.go
+2
-2
repo/repo.go
···
91
91
bufr := bufrPool.Get().(*bufio.Reader)
92
92
bufr.Reset(r)
93
93
94
+
defer bufrPool.Put(bufr)
95
+
94
96
br, root, err := carutil.NewReader(bufr)
95
97
if err != nil {
96
98
return cid.Undef, fmt.Errorf("opening CAR block reader: %w", err)
···
109
111
return cid.Undef, fmt.Errorf("copying block to store: %w", err)
110
112
}
111
113
}
112
-
113
-
bufrPool.Put(bufr)
114
114
115
115
return root, nil
116
116
}
+3
-7
repo/stream.go
+3
-7
repo/stream.go
···
16
16
"go.opentelemetry.io/otel"
17
17
)
18
18
19
-
const bufPoolBlockSize = 512
19
+
const bufPoolBlockSize = 1024
20
20
21
21
var smallBlockPool = &sync.Pool{
22
22
New: func() any {
···
29
29
streamComplete bool
30
30
31
31
r *carutil.Reader
32
-
33
-
outOfOrder int
34
-
totalBlocks int
35
32
}
36
33
37
34
func newStreamingBlockstore(r *carutil.Reader) *readStreamBlockstore {
···
57
54
smallBlockPool.Put(buf)
58
55
}
59
56
60
-
bs.totalBlocks++
61
57
if blk.Cid() == cc {
62
58
return blk, nil
63
59
}
64
60
65
-
bs.outOfOrder++
66
61
bs.otherBlocks[blk.Cid()] = blk
67
62
}
68
63
···
142
137
143
138
t := mst.LoadMST(cst, sc.Data)
144
139
140
+
t.SetNoCache(true)
141
+
145
142
if err := t.WalkLeavesFrom(ctx, prefix, func(k string, val cid.Cid) error {
146
143
if err := bs.View(val, func(data []byte) error {
147
144
return cb(k, val, data)
···
155
152
return "", fmt.Errorf("failed to walk mst: %w", err)
156
153
}
157
154
158
-
fmt.Println("out of order blocks: ", bs.outOfOrder, bs.totalBlocks)
159
155
return sc.Rev, nil
160
156
}