this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

flag in mst code to avoid caching internal stuff, makes oncethrough iteration nicer

authored by

whyrusleeping and committed by
whyrusleeping
95c64e97 a5f57b26

+30 -25
+20 -11
mst/mst.go
··· 137 137 layer int 138 138 pointer cid.Cid 139 139 validPtr bool 140 + 141 + noCache bool 140 142 } 141 143 142 144 // NewEmptyMST reports a new empty MST using cst as its storage. 143 145 func NewEmptyMST(cst cbor.IpldStore) *MerkleSearchTree { 144 - return createMST(cst, cid.Undef, []nodeEntry{}, 0) 146 + return createMST(cst, cid.Undef, []nodeEntry{}, 0, false) 145 147 } 146 148 147 149 // Typescript: MST.create(storage, entries, layer, fanout) -> MST 148 - func createMST(cst cbor.IpldStore, ptr cid.Cid, entries []nodeEntry, layer int) *MerkleSearchTree { 150 + func createMST(cst cbor.IpldStore, ptr cid.Cid, entries []nodeEntry, layer int, noCache bool) *MerkleSearchTree { 149 151 mst := &MerkleSearchTree{ 150 152 cst: cst, 151 153 pointer: ptr, 152 154 layer: layer, 153 155 entries: entries, 154 156 validPtr: ptr.Defined(), 157 + noCache: noCache, 155 158 } 156 159 157 160 return mst ··· 162 165 // This is poorly named in both implementations, because it is lazy 163 166 // Typescript: MST.load(storage, cid, layer=null, fanout) -> MST 164 167 func LoadMST(cst cbor.IpldStore, root cid.Cid) *MerkleSearchTree { 165 - return createMST(cst, root, nil, -1) 168 + return createMST(cst, root, nil, -1, false) 169 + } 170 + 171 + func (mst *MerkleSearchTree) SetNoCache(v bool) { 172 + mst.noCache = v 166 173 } 167 174 168 175 // === "Immutability" === ··· 173 180 if entries == nil { 174 181 panic("nil entries passed to newTree") 175 182 } 176 - return createMST(mst.cst, cid.Undef, entries, mst.layer) 183 + return createMST(mst.cst, cid.Undef, entries, mst.layer, false) 177 184 } 178 185 179 186 // === "Getters (lazy load)" === ··· 195 202 } 196 203 // NOTE(bnewbold): Typescript version computes layer in-place here, but 197 204 // the entriesFromNodeData() helper does that for us in golang 198 - entries, err := entriesFromNodeData(ctx, &nd, mst.cst) 205 + entries, err := entriesFromNodeData(ctx, &nd, mst.cst, mst.noCache) 199 206 if err != nil { 200 207 return nil, err 201 208 } 202 209 if entries == nil { 203 210 panic("got nil entries from node data decoding") 204 211 } 205 - mst.entries = entries 212 + if !mst.noCache { 213 + mst.entries = entries 214 + } 206 215 return entries, nil 207 216 } 208 217 ··· 210 219 } 211 220 212 221 // golang-specific helper that calls in to deserializeNodeData 213 - func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore) ([]nodeEntry, error) { 222 + func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore, noCache bool) ([]nodeEntry, error) { 214 223 layer := -1 215 224 if len(nd.Entries) > 0 { 216 225 // NOTE(bnewbold): can compute the layer on the first KeySuffix, because for the first entry that field is a complete key ··· 218 227 layer = leadingZerosOnHashBytes(firstLeaf.KeySuffix) 219 228 } 220 229 221 - entries, err := deserializeNodeData(ctx, cst, nd, layer) 230 + entries, err := deserializeNodeData(ctx, cst, nd, layer, noCache) 222 231 if err != nil { 223 232 return nil, err 224 233 } ··· 468 477 } 469 478 470 479 checkTreeInvariant(updated) 471 - newRoot := createMST(mst.cst, cid.Undef, updated, keyZeros) 480 + newRoot := createMST(mst.cst, cid.Undef, updated, keyZeros, mst.noCache) 472 481 473 482 // NOTE(bnewbold): We do want to invalid the CID (because this node has 474 483 // changed, and we are "lazy" about recomputing). Setting this flag ··· 898 907 return nil, err 899 908 } 900 909 901 - return createMST(mst.cst, cid.Undef, []nodeEntry{}, layer-1), nil 910 + return createMST(mst.cst, cid.Undef, []nodeEntry{}, layer-1, mst.noCache), nil 902 911 } 903 912 904 913 func (mst *MerkleSearchTree) createParent(ctx context.Context) (*MerkleSearchTree, error) { ··· 907 916 return nil, err 908 917 } 909 918 910 - return createMST(mst.cst, cid.Undef, []nodeEntry{mkTreeEntry(mst)}, layer+1), nil 919 + return createMST(mst.cst, cid.Undef, []nodeEntry{mkTreeEntry(mst)}, layer+1, mst.noCache), nil 911 920 } 912 921 913 922 // === "Finding insertion points" ===
+2 -2
mst/mst_test.go
··· 36 36 37 37 ctx := context.Background() 38 38 cst := util.CborStore(blockstore.NewBlockstore(datastore.NewMapDatastore())) 39 - mst := createMST(cst, cid.Undef, []nodeEntry{}, -1) 39 + mst := createMST(cst, cid.Undef, []nodeEntry{}, -1, false) 40 40 41 41 // NOTE: these were previously generated randomly, but the random seed behavior changed 42 42 vals := map[string]cid.Cid{ ··· 364 364 365 365 func cidMapToMst(t testing.TB, bs blockstore.Blockstore, m map[string]cid.Cid) *MerkleSearchTree { 366 366 cst := util.CborStore(bs) 367 - mt := createMST(cst, cid.Undef, []nodeEntry{}, -1) 367 + mt := createMST(cst, cid.Undef, []nodeEntry{}, -1, false) 368 368 369 369 for k, v := range m { 370 370 nmst, err := mt.Add(context.TODO(), k, v, -1)
+3 -3
mst/mst_util.go
··· 66 66 } 67 67 68 68 // Typescript: deserializeNodeData(storage, data, layer) 69 - func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int) ([]nodeEntry, error) { 69 + func deserializeNodeData(ctx context.Context, cst cbor.IpldStore, nd *NodeData, layer int, noCache bool) ([]nodeEntry, error) { 70 70 entries := []nodeEntry{} 71 71 if nd.Left != nil { 72 72 // Note: like Typescript, this is actually a lazy load 73 73 entries = append(entries, nodeEntry{ 74 74 Kind: entryTree, 75 - Tree: createMST(cst, *nd.Left, nil, layer-1), 75 + Tree: createMST(cst, *nd.Left, nil, layer-1, noCache), 76 76 }) 77 77 } 78 78 ··· 100 100 if e.Tree != nil { 101 101 entries = append(entries, nodeEntry{ 102 102 Kind: entryTree, 103 - Tree: createMST(cst, *e.Tree, nil, layer-1), 103 + Tree: createMST(cst, *e.Tree, nil, layer-1, noCache), 104 104 Key: keyStr, 105 105 }) 106 106 }
+2 -2
repo/repo.go
··· 91 91 bufr := bufrPool.Get().(*bufio.Reader) 92 92 bufr.Reset(r) 93 93 94 + defer bufrPool.Put(bufr) 95 + 94 96 br, root, err := carutil.NewReader(bufr) 95 97 if err != nil { 96 98 return cid.Undef, fmt.Errorf("opening CAR block reader: %w", err) ··· 109 111 return cid.Undef, fmt.Errorf("copying block to store: %w", err) 110 112 } 111 113 } 112 - 113 - bufrPool.Put(bufr) 114 114 115 115 return root, nil 116 116 }
+3 -7
repo/stream.go
··· 16 16 "go.opentelemetry.io/otel" 17 17 ) 18 18 19 - const bufPoolBlockSize = 512 19 + const bufPoolBlockSize = 1024 20 20 21 21 var smallBlockPool = &sync.Pool{ 22 22 New: func() any { ··· 29 29 streamComplete bool 30 30 31 31 r *carutil.Reader 32 - 33 - outOfOrder int 34 - totalBlocks int 35 32 } 36 33 37 34 func newStreamingBlockstore(r *carutil.Reader) *readStreamBlockstore { ··· 57 54 smallBlockPool.Put(buf) 58 55 } 59 56 60 - bs.totalBlocks++ 61 57 if blk.Cid() == cc { 62 58 return blk, nil 63 59 } 64 60 65 - bs.outOfOrder++ 66 61 bs.otherBlocks[blk.Cid()] = blk 67 62 } 68 63 ··· 142 137 143 138 t := mst.LoadMST(cst, sc.Data) 144 139 140 + t.SetNoCache(true) 141 + 145 142 if err := t.WalkLeavesFrom(ctx, prefix, func(k string, val cid.Cid) error { 146 143 if err := bs.View(val, func(data []byte) error { 147 144 return cb(k, val, data) ··· 155 152 return "", fmt.Errorf("failed to walk mst: %w", err) 156 153 } 157 154 158 - fmt.Println("out of order blocks: ", bs.outOfOrder, bs.totalBlocks) 159 155 return sc.Rev, nil 160 156 }