+4 -1

cmd/gosky/main.go

reviewed

··· 359 359 } 360 360 361 361 user := cctx.Args().First() 362 362 + if user == "" { 363 363 + user = bskyc.C.Auth.Did 364 364 + } 362 365 363 366 ctx := context.TODO() 364 367 resp, err := bskyc.GraphGetFollows(ctx, user, 100, nil) ··· 367 370 } 368 371 369 372 for _, f := range resp.Follows { 370 370 - fmt.Println(f.Did) 373 373 + fmt.Println(f.Did, f.Handle) 371 374 } 372 375 373 376 return nil

+3 -3

mst/cbor_gen.go

reviewed

··· 263 263 return err 264 264 } 265 265 266 266 - if t.T == nil { 266 266 + if t.Tree == nil { 267 267 if _, err := cw.Write(cbg.CborNull); err != nil { 268 268 return err 269 269 } 270 270 } else { 271 271 - if err := cbg.WriteCid(cw, *t.T); err != nil { 271 271 + if err := cbg.WriteCid(cw, *t.Tree); err != nil { 272 272 return xerrors.Errorf("failed to write cid field t.T: %w", err) 273 273 } 274 274 } ··· 383 383 return xerrors.Errorf("failed to read cid field t.T: %w", err) 384 384 } 385 385 386 386 - t.T = &c 386 386 + t.Tree = &c 387 387 } 388 388 389 389 }

+109 -64

mst/mst.go

reviewed

··· 39 39 } 40 40 41 41 func LoadMST(cst cbor.IpldStore, fanout int, root cid.Cid) *MerkleSearchTree { 42 42 - return NewMST(cst, fanout, root, nil, 0) 42 42 + return NewMST(cst, fanout, root, nil, -1) 43 43 } 44 44 45 45 func (mst *MerkleSearchTree) newTree(entries []NodeEntry) *MerkleSearchTree { ··· 78 78 } 79 79 80 80 func create(ctx context.Context, cst cbor.IpldStore, entries []NodeEntry, layer int, fanout int) (*MerkleSearchTree, error) { 81 81 - ptr, err := cidForEntries(ctx, entries, cst) 82 82 - if err != nil { 83 83 - return nil, err 84 84 - } 85 85 - 81 81 + var ptr cid.Cid 86 82 return NewMST(cst, fanout, ptr, entries, layer), nil 87 83 } 88 84 ··· 132 128 } 133 129 134 130 type TreeEntry struct { 135 135 - P int64 `cborgen:"p"` 136 136 - K string `cborgen:"k"` 137 137 - V cid.Cid `cborgen:"v"` 138 138 - T *cid.Cid `cborgen:"t"` 131 131 + P int64 `cborgen:"p"` 132 132 + K string `cborgen:"k"` 133 133 + V cid.Cid `cborgen:"v"` 134 134 + Tree *cid.Cid `cborgen:"t"` 139 135 } 140 136 141 137 type NodeData struct { ··· 151 147 152 148 layer, err := mst.getLayer(ctx) 153 149 if err != nil { 154 154 - return nil, err 150 150 + return nil, fmt.Errorf("getting layer failed: %w", err) 155 151 } 156 152 157 153 newLeaf := NodeEntry{ ··· 191 187 } 192 188 193 189 return mst.replaceWithSplit(ctx, index-1, left, newLeaf, right) 194 194 - } else if keyZeros > layer { 190 190 + } else if keyZeros < layer { 195 191 index, err := mst.findGtOrEqualLeafIndex(ctx, key) 196 192 if err != nil { 197 193 return nil, err ··· 208 204 return nil, err 209 205 } 210 206 211 211 - return mst.updateEntry(ctx, index-1, NodeEntry{ 212 212 - Kind: EntryTree, 213 213 - Tree: newSubtree, 214 214 - }) 207 207 + return mst.updateEntry(ctx, index-1, treeEntry(newSubtree)) 215 208 } else { 216 209 subTree, err := mst.createChild(ctx) 217 210 if err != nil { ··· 220 213 221 214 newSubTree, err := subTree.Add(ctx, key, val, keyZeros) 222 215 if err != nil { 223 223 - return nil, err 216 216 + return nil, fmt.Errorf("subtree add: %w", err) 224 217 } 225 218 226 219 return mst.spliceIn(ctx, treeEntry(newSubTree), index) ··· 233 226 234 227 layer, err := mst.getLayer(ctx) 235 228 if err != nil { 236 236 - return nil, err 229 229 + return nil, fmt.Errorf("get layer in split case failed: %w", err) 237 230 } 238 231 239 232 extraLayersToAdd := keyZeros - layer ··· 243 236 if left != nil { 244 237 par, err := left.createParent(ctx) 245 238 if err != nil { 246 246 - return nil, err 239 239 + return nil, fmt.Errorf("create left parent: %w", err) 247 240 } 248 241 left = par 249 242 } ··· 251 244 if right != nil { 252 245 par, err := right.createParent(ctx) 253 246 if err != nil { 254 254 - return nil, err 247 247 + return nil, fmt.Errorf("create right parent: %w", err) 255 248 } 256 249 right = par 257 250 } 258 251 259 252 } 253 253 + 260 254 var updated []NodeEntry 261 255 if left != nil { 262 256 updated = append(updated, treeEntry(left)) 263 257 } 258 258 + 259 259 + updated = append(updated, NodeEntry{ 260 260 + Kind: EntryLeaf, 261 261 + Key: key, 262 262 + Val: val, 263 263 + }) 264 264 + 264 265 if right != nil { 265 266 updated = append(updated, treeEntry(right)) 266 267 } 267 268 269 269 + checkTreeInvariant(updated) 268 270 newRoot, err := create(ctx, mst.cst, updated, keyZeros, mst.fanout) 269 271 if err != nil { 270 270 - return nil, err 272 272 + return nil, fmt.Errorf("creating new tree after split: %w", err) 271 273 } 274 274 + 272 275 // why invalidate? 273 276 newRoot.validPtr = false 274 277 ··· 316 319 return nil, err 317 320 } 318 321 319 319 - return NewMST(mst.cst, mst.fanout, cid.Undef, nil, layer-1), nil 322 322 + return NewMST(mst.cst, mst.fanout, cid.Undef, []NodeEntry{}, layer-1), nil 320 323 } 321 324 322 325 func (mst *MerkleSearchTree) updateEntry(ctx context.Context, ix int, entry NodeEntry) (*MerkleSearchTree, error) { ··· 330 333 nents[ix] = entry 331 334 copy(nents[ix+1:], entries[ix+1:]) 332 335 336 336 + checkTreeInvariant(nents) 337 337 + 333 338 return mst.newTree(nents), nil 334 339 } 335 340 ··· 338 343 if err != nil { 339 344 return nil, err 340 345 } 346 346 + checkTreeInvariant(entries) 341 347 var update []NodeEntry 342 348 update = append(update, entries[:ix]...) 343 349 ··· 357 363 }) 358 364 } 359 365 360 360 - update = append(update, entries[ix:]...) 366 366 + update = append(update, entries[ix+1:]...) 361 367 368 368 + checkTreeInvariant(update) 362 369 return mst.newTree(update), nil 363 370 } 364 371 372 372 + func checkTreeInvariant(ents []NodeEntry) { 373 373 + for i := 0; i < len(ents)-1; i++ { 374 374 + if ents[i].isTree() && ents[i+1].isTree() { 375 375 + panic(fmt.Sprintf("two trees next to eachother! %d %d", i, i+1)) 376 376 + } 377 377 + } 378 378 + } 379 379 + 365 380 func (mst *MerkleSearchTree) splitAround(ctx context.Context, key string) (*MerkleSearchTree, *MerkleSearchTree, error) { 366 381 index, err := mst.findGtOrEqualLeafIndex(ctx, key) 367 382 if err != nil { ··· 392 407 return nil, nil, err 393 408 } 394 409 395 395 - left, err = left.append(ctx, treeEntry(subl)) 396 396 - if err != nil { 397 397 - return nil, nil, err 410 410 + if subl != nil { 411 411 + left, err = left.append(ctx, treeEntry(subl)) 412 412 + if err != nil { 413 413 + return nil, nil, err 414 414 + } 398 415 } 399 416 400 400 - right, err = right.prepend(ctx, treeEntry(subr)) 401 401 - if err != nil { 402 402 - return nil, nil, err 417 417 + if subr != nil { 418 418 + right, err = right.prepend(ctx, treeEntry(subr)) 419 419 + if err != nil { 420 420 + return nil, nil, err 421 421 + } 403 422 } 404 404 - 405 423 } 406 424 407 425 if left.entryCount() == 0 { ··· 410 428 if right.entryCount() == 0 { 411 429 right = nil 412 430 } 431 431 + 413 432 return left, right, nil 414 433 } 415 434 ··· 432 451 copy(nents, entries) 433 452 nents[len(nents)-1] = ent 434 453 454 454 + checkTreeInvariant(nents) 435 455 return mst.newTree(nents), nil 436 456 } 437 457 ··· 445 465 copy(nents[1:], entries) 446 466 nents[0] = ent 447 467 468 468 + checkTreeInvariant(nents) 448 469 return mst.newTree(nents), nil 449 470 } 450 471 ··· 457 478 nents := make([]NodeEntry, len(entries)-1) 458 479 copy(nents, entries[:ix]) 459 480 copy(nents[ix:], entries[ix+1:]) 481 481 + checkTreeInvariant(nents) 460 482 return mst.newTree(nents), nil 461 483 } 462 484 ··· 471 493 nents[ix] = entry 472 494 copy(nents[ix+1:], entries[ix:]) 473 495 496 496 + checkTreeInvariant(nents) 474 497 return mst.newTree(nents), nil 475 498 } 476 499 ··· 515 538 return nil, err 516 539 } 517 540 518 518 - if len(nd.E) == 0 { 519 519 - // maybe this should be an error? idk 520 520 - return nil, nil 521 521 - } 522 522 - 523 523 - firstLeaf := nd.E[0] 524 524 - 525 525 - layer := leadingZerosOnHash(firstLeaf.K, mst.fanout) 526 526 - 527 527 - entries, err := deserializeNodeData(ctx, mst.cst, &nd, layer, mst.fanout) 541 541 + entries, err := entriesFromNodeData(ctx, &nd, mst.cst, mst.fanout) 528 542 if err != nil { 529 543 return nil, err 530 544 } 531 531 - 532 545 mst.entries = entries 533 546 return entries, nil 534 547 } ··· 536 549 return nil, fmt.Errorf("no entries or cid provided") 537 550 } 538 551 552 552 + func entriesFromNodeData(ctx context.Context, nd *NodeData, cst cbor.IpldStore, fanout int) ([]NodeEntry, error) { 553 553 + layer := -1 554 554 + if len(nd.E) > 0 { 555 555 + firstLeaf := nd.E[0] 556 556 + layer = leadingZerosOnHash(firstLeaf.K, fanout) 557 557 + } 558 558 + 559 559 + entries, err := deserializeNodeData(ctx, cst, nd, layer, fanout) 560 560 + if err != nil { 561 561 + return nil, err 562 562 + } 563 563 + 564 564 + return entries, nil 565 565 + } 566 566 + 539 567 func (mst *MerkleSearchTree) getPointer(ctx context.Context) (cid.Cid, error) { 540 568 if mst.validPtr { 541 569 return mst.pointer, nil ··· 582 610 func cidForEntries(ctx context.Context, entries []NodeEntry, cst cbor.IpldStore) (cid.Cid, error) { 583 611 nd, err := serializeNodeData(entries) 584 612 if err != nil { 585 585 - return cid.Undef, err 613 613 + return cid.Undef, fmt.Errorf("serializing new entries: %w", err) 586 614 } 587 615 588 616 return cst.Put(ctx, nd) ··· 595 623 if len(entries) > 0 && entries[0].isTree() { 596 624 i++ 597 625 598 598 - if !entries[0].Tree.validPtr { 599 599 - panic("invalid pointer in entries list tree") 626 626 + ptr, err := entries[0].Tree.GetPointer(context.TODO()) 627 627 + if err != nil { 628 628 + return nil, err 600 629 } 601 601 - 602 602 - ptr := entries[0].Tree.pointer 603 630 data.L = &ptr 604 631 } 605 632 ··· 608 635 leaf := entries[i] 609 636 610 637 if !leaf.isLeaf() { 611 611 - return nil, fmt.Errorf("Not a valid node: two subtrees next to eachother") 638 638 + return nil, fmt.Errorf("Not a valid node: two subtrees next to eachother (%d, %d)", i, len(entries)) 612 639 } 613 640 i++ 614 641 ··· 618 645 next := entries[i] 619 646 620 647 if next.isTree() { 621 621 - subtree = &next.Tree.pointer 622 622 - i++ 623 623 - } 624 648 625 625 - prefixLen := countPrefixLen(lastKey, leaf.Key) 626 626 - data.E = append(data.E, TreeEntry{ 627 627 - P: int64(prefixLen), 628 628 - K: leaf.Key[prefixLen:], 629 629 - V: leaf.Val, 630 630 - T: subtree, 631 631 - }) 649 649 + ptr, err := next.Tree.GetPointer(context.TODO()) 650 650 + if err != nil { 651 651 + return nil, fmt.Errorf("getting subtree pointer: %w", err) 652 652 + } 632 653 654 654 + subtree = &ptr 655 655 + i++ 656 656 + } 633 657 } 658 658 + 659 659 + prefixLen := countPrefixLen(lastKey, leaf.Key) 660 660 + data.E = append(data.E, TreeEntry{ 661 661 + P: int64(prefixLen), 662 662 + K: leaf.Key[prefixLen:], 663 663 + V: leaf.Val, 664 664 + Tree: subtree, 665 665 + }) 634 666 635 667 lastKey = leaf.Key 636 668 } ··· 670 702 Val: e.V, 671 703 }) 672 704 673 673 - lastKey = string(key) 674 674 - if e.T != nil { 705 705 + if e.Tree != nil { 675 706 entries = append(entries, NodeEntry{ 676 707 Kind: EntryTree, 677 677 - Tree: NewMST(cst, fanout, *e.T, nil, layer-1), 708 708 + Tree: NewMST(cst, fanout, *e.Tree, nil, layer-1), 709 709 + Key: string(key), 678 710 }) 679 711 } 712 712 + lastKey = string(key) 680 713 } 681 714 682 715 return entries, nil ··· 700 733 } 701 734 702 735 func (mst *MerkleSearchTree) getLayer(ctx context.Context) (int, error) { 703 703 - if mst.layer != -1 { 736 736 + if mst.layer >= 0 { 704 737 return mst.layer, nil 705 738 } 706 739 ··· 710 743 } 711 744 712 745 mst.layer = layerForEntries(entries, mst.fanout) 746 746 + if mst.layer < 0 { 747 747 + // still empty! 748 748 + mst.layer = 0 749 749 + } 750 750 + 713 751 return mst.layer, nil 714 752 } 715 753 754 754 + func log2(v int) int { 755 755 + var out int 756 756 + for v > 1 { 757 757 + out++ 758 758 + v = v / 2 759 759 + } 760 760 + return out 761 761 + } 762 762 + 716 763 func leadingZerosOnHash(k string, fanout int) int { 717 764 hv := sha256.Sum256([]byte(k)) 718 765 ··· 724 771 break 725 772 } 726 773 } 727 727 - return total / fanout 774 774 + return total / log2(fanout) 728 775 } 729 776 730 777 func (mst *MerkleSearchTree) WalkLeavesFrom(ctx context.Context, key string, cb func(n NodeEntry) error) error { ··· 737 784 if err != nil { 738 785 return err 739 786 } 740 740 - 741 741 - fmt.Println(entries) 742 787 743 788 if index > 0 { 744 789 prev := entries[index-1]

+405 -2

mst/mst_test.go

reviewed

··· 2 2 3 3 import ( 4 4 "context" 5 5 - "crypto/rand" 5 5 + "encoding/hex" 6 6 "fmt" 7 7 + "io" 8 8 + "math/rand" 9 9 + "os" 10 10 + "sort" 7 11 "testing" 8 12 9 13 cid "github.com/ipfs/go-cid" 10 14 "github.com/ipfs/go-datastore" 11 15 blockstore "github.com/ipfs/go-ipfs-blockstore" 12 16 cbor "github.com/ipfs/go-ipld-cbor" 17 17 + "github.com/ipld/go-car/v2" 18 18 + "github.com/multiformats/go-multihash" 13 19 mh "github.com/multiformats/go-multihash" 14 20 ) 15 21 ··· 26 32 func TestBasicMst(t *testing.T) { 27 33 ctx := context.Background() 28 34 cst := cbor.NewCborStore(blockstore.NewBlockstore(datastore.NewMapDatastore())) 29 29 - mst := NewMST(cst, 16, cid.Undef, []NodeEntry{}, 0) 35 35 + mst := NewMST(cst, 16, cid.Undef, []NodeEntry{}, -1) 30 36 31 37 vals := map[string]cid.Cid{ 32 38 "cats": randCid(), ··· 49 55 50 56 fmt.Println(ncid) 51 57 } 58 58 + 59 59 + func TestEdgeCase9(t *testing.T) { 60 60 + m := map[string]string{ 61 61 + "97206d5e4a18/19fbf0b79789/1710133f2dd6": "cats", 62 62 + } 63 63 + 64 64 + bs := memBs() 65 65 + mst := cidMapToMst(t, 32, bs, mapToCidMap(m)) 66 66 + _ = mst 67 67 + 68 68 + } 69 69 + 70 70 + func mustCid(t *testing.T, s string) cid.Cid { 71 71 + t.Helper() 72 72 + c, err := cid.Decode(s) 73 73 + if err != nil { 74 74 + t.Fatal(err) 75 75 + } 76 76 + return c 77 77 + 78 78 + } 79 79 + 80 80 + func loadCar(bs blockstore.Blockstore, fname string) error { 81 81 + fi, err := os.Open(fname) 82 82 + if err != nil { 83 83 + return err 84 84 + } 85 85 + br, err := car.NewBlockReader(fi) 86 86 + if err != nil { 87 87 + return err 88 88 + } 89 89 + 90 90 + for { 91 91 + blk, err := br.Next() 92 92 + if err != nil { 93 93 + if err == io.EOF { 94 94 + break 95 95 + } 96 96 + return err 97 97 + } 98 98 + 99 99 + if err := bs.Put(context.TODO(), blk); err != nil { 100 100 + return err 101 101 + } 102 102 + } 103 103 + 104 104 + return nil 105 105 + } 106 106 + 107 107 + func TestDiff(t *testing.T) { 108 108 + to := mustCid(t, "bafyreie5cvv4h45feadgeuwhbcutmh6t2ceseocckahdoe6uat64zmz454") 109 109 + from := mustCid(t, "bafyreigv5er7vcxlbikkwedmtd7b3kp7wrcyffep5ogcuxosloxfox5reu") 110 110 + 111 111 + bs := blockstore.NewBlockstore(datastore.NewMapDatastore()) 112 112 + 113 113 + if err := loadCar(bs, "paul.car"); err != nil { 114 114 + t.Fatal(err) 115 115 + } 116 116 + 117 117 + ctx := context.TODO() 118 118 + ops, err := DiffTrees(ctx, bs, from, to) 119 119 + if err != nil { 120 120 + t.Fatal(err) 121 121 + } 122 122 + _ = ops 123 123 + } 124 124 + 125 125 + func randStr(s int64) string { 126 126 + buf := make([]byte, 6) 127 127 + r := rand.New(rand.NewSource(s)) 128 128 + r.Read(buf) 129 129 + return hex.EncodeToString(buf) 130 130 + } 131 131 + 132 132 + func copyMap(a map[string]string) map[string]string { 133 133 + out := make(map[string]string) 134 134 + for k, v := range a { 135 135 + out[k] = v 136 136 + } 137 137 + return out 138 138 + } 139 139 + 140 140 + func TestDiffInsertionsBasic(t *testing.T) { 141 141 + a := map[string]string{ 142 142 + "cats/asdf": randStr(1), 143 143 + "cats/foosesdf": randStr(2), 144 144 + } 145 145 + 146 146 + b := copyMap(a) 147 147 + b["cats/bawda"] = randStr(3) 148 148 + b["cats/crosasd"] = randStr(4) 149 149 + 150 150 + testMapDiffs(t, a, b, 32) 151 151 + testMapDiffs(t, b, a, 32) 152 152 + } 153 153 + 154 154 + func randKey(s int64) string { 155 155 + r := rand.New(rand.NewSource(s)) 156 156 + 157 157 + top := r.Int63n(6) 158 158 + mid := r.Int63n(3) 159 159 + 160 160 + end := randStr(r.Int63n(10000000)) 161 161 + 162 162 + return randStr(125125+top) + "/" + randStr(858392+mid) + "/" + end 163 163 + } 164 164 + 165 165 + func TestDiffInsertionsLarge(t *testing.T) { 166 166 + a := map[string]string{} 167 167 + for i := int64(0); i < 1000; i++ { 168 168 + a[randKey(i)] = randStr(72385739 - i) 169 169 + } 170 170 + 171 171 + b := copyMap(a) 172 172 + for i := int64(0); i < 30; i++ { 173 173 + b[randKey(5000+i)] = randStr(2293825 - i) 174 174 + } 175 175 + 176 176 + testMapDiffs(t, a, b, 32) 177 177 + testMapDiffs(t, b, a, 32) 178 178 + } 179 179 + 180 180 + func TestDiffNoOverlap(t *testing.T) { 181 181 + a := map[string]string{} 182 182 + for i := int64(0); i < 10; i++ { 183 183 + a[randKey(i)] = randStr(72385739 - i) 184 184 + } 185 185 + 186 186 + b := map[string]string{} 187 187 + for i := int64(0); i < 10; i++ { 188 188 + b[randKey(5000+i)] = randStr(2293825 - i) 189 189 + } 190 190 + 191 191 + testMapDiffs(t, a, b, 32) 192 192 + testMapDiffs(t, b, a, 32) 193 193 + } 194 194 + 195 195 + func TestDiffSmallOverlap(t *testing.T) { 196 196 + a := map[string]string{} 197 197 + for i := int64(0); i < 10; i++ { 198 198 + a[randKey(i)] = randStr(72385739 - i) 199 199 + } 200 200 + 201 201 + b := copyMap(a) 202 202 + 203 203 + for i := int64(0); i < 1000; i++ { 204 204 + a[randKey(i)] = randStr(682823 - i) 205 205 + } 206 206 + 207 207 + for i := int64(0); i < 1000; i++ { 208 208 + b[randKey(5000+i)] = randStr(2293825 - i) 209 209 + } 210 210 + 211 211 + testMapDiffs(t, a, b, 32) 212 212 + //testMapDiffs(t, b, a) 213 213 + } 214 214 + 215 215 + func TestDiffSmallOverlapSmall(t *testing.T) { 216 216 + a := map[string]string{} 217 217 + for i := int64(0); i < 4; i++ { 218 218 + a[randKey(i)] = randStr(72385739 - i) 219 219 + } 220 220 + 221 221 + b := copyMap(a) 222 222 + 223 223 + for i := int64(0); i < 20; i++ { 224 224 + a[randKey(i)] = randStr(682823 - i) 225 225 + } 226 226 + 227 227 + for i := int64(0); i < 20; i++ { 228 228 + b[randKey(5000+i)] = randStr(2293825 - i) 229 229 + } 230 230 + 231 231 + testMapDiffs(t, a, b, 4) 232 232 + //testMapDiffs(t, b, a) 233 233 + } 234 234 + 235 235 + func TestDiffMutationsBasic(t *testing.T) { 236 236 + a := map[string]string{ 237 237 + "cats/asdf": randStr(1), 238 238 + "cats/foosesdf": randStr(2), 239 239 + } 240 240 + 241 241 + b := copyMap(a) 242 242 + b["cats/asdf"] = randStr(3) 243 243 + 244 244 + testMapDiffs(t, a, b, 32) 245 245 + } 246 246 + 247 247 + func diffMaps(a, b map[string]cid.Cid) []*DiffOp { 248 248 + var akeys, bkeys []string 249 249 + 250 250 + for k := range a { 251 251 + akeys = append(akeys, k) 252 252 + } 253 253 + 254 254 + for k := range b { 255 255 + bkeys = append(bkeys, k) 256 256 + } 257 257 + 258 258 + sort.Strings(akeys) 259 259 + sort.Strings(bkeys) 260 260 + 261 261 + var out []*DiffOp 262 262 + for _, k := range akeys { 263 263 + av := a[k] 264 264 + bv, ok := b[k] 265 265 + if !ok { 266 266 + out = append(out, &DiffOp{ 267 267 + Op: "del", 268 268 + Tid: k, 269 269 + OldCid: av, 270 270 + }) 271 271 + } else { 272 272 + if av != bv { 273 273 + out = append(out, &DiffOp{ 274 274 + Op: "mut", 275 275 + Tid: k, 276 276 + OldCid: av, 277 277 + NewCid: bv, 278 278 + }) 279 279 + } 280 280 + } 281 281 + } 282 282 + 283 283 + for _, k := range bkeys { 284 284 + _, ok := a[k] 285 285 + if !ok { 286 286 + out = append(out, &DiffOp{ 287 287 + Op: "add", 288 288 + Tid: k, 289 289 + NewCid: b[k], 290 290 + }) 291 291 + } 292 292 + } 293 293 + 294 294 + sort.Slice(out, func(i, j int) bool { 295 295 + return out[i].Tid < out[j].Tid 296 296 + }) 297 297 + 298 298 + return out 299 299 + } 300 300 + 301 301 + func strToCid(s string) cid.Cid { 302 302 + h, err := multihash.Sum([]byte(s), multihash.ID, -1) 303 303 + if err != nil { 304 304 + panic(err) 305 305 + } 306 306 + 307 307 + return cid.NewCidV1(cid.Raw, h) 308 308 + 309 309 + } 310 310 + 311 311 + func mapToCidMap(a map[string]string) map[string]cid.Cid { 312 312 + out := make(map[string]cid.Cid) 313 313 + for k, v := range a { 314 314 + out[k] = strToCid(v) 315 315 + } 316 316 + 317 317 + return out 318 318 + } 319 319 + 320 320 + func cidMapToMst(t *testing.T, fanout int, bs blockstore.Blockstore, m map[string]cid.Cid) *MerkleSearchTree { 321 321 + cst := cbor.NewCborStore(bs) 322 322 + mt := NewMST(cst, fanout, cid.Undef, []NodeEntry{}, -1) 323 323 + 324 324 + for k, v := range m { 325 325 + nmst, err := mt.Add(context.TODO(), k, v, -1) 326 326 + if err != nil { 327 327 + t.Fatal(err) 328 328 + } 329 329 + 330 330 + mt = nmst 331 331 + } 332 332 + 333 333 + return mt 334 334 + } 335 335 + 336 336 + func mustCidTree(t *testing.T, tree *MerkleSearchTree) cid.Cid { 337 337 + c, err := tree.GetPointer(context.TODO()) 338 338 + if err != nil { 339 339 + t.Fatal(err) 340 340 + } 341 341 + return c 342 342 + } 343 343 + 344 344 + func memBs() blockstore.Blockstore { 345 345 + return blockstore.NewBlockstore(datastore.NewMapDatastore()) 346 346 + } 347 347 + 348 348 + func testMapDiffs(t *testing.T, a, b map[string]string, fanout int) { 349 349 + amc := mapToCidMap(a) 350 350 + bmc := mapToCidMap(b) 351 351 + 352 352 + exp := diffMaps(amc, bmc) 353 353 + 354 354 + bs := memBs() 355 355 + 356 356 + msta := cidMapToMst(t, fanout, bs, amc) 357 357 + mstb := cidMapToMst(t, fanout, bs, bmc) 358 358 + 359 359 + cida := mustCidTree(t, msta) 360 360 + cidb := mustCidTree(t, mstb) 361 361 + 362 362 + diffs, err := DiffTrees(context.TODO(), bs, cida, cidb) 363 363 + if err != nil { 364 364 + t.Fatal(err) 365 365 + } 366 366 + 367 367 + if !sort.SliceIsSorted(diffs, func(i, j int) bool { 368 368 + return diffs[i].Tid < diffs[j].Tid 369 369 + }) { 370 370 + t.Log("diff algo did not produce properly sorted diff") 371 371 + } 372 372 + if !compareDiffs(diffs, exp) { 373 373 + fmt.Println("Expected Diff:") 374 374 + for _, do := range exp { 375 375 + fmt.Println(do) 376 376 + } 377 377 + fmt.Println("Actual Diff:") 378 378 + for _, do := range diffs { 379 379 + fmt.Println(do) 380 380 + } 381 381 + t.Logf("diff lens: %d %d", len(diffs), len(exp)) 382 382 + diffDiff(diffs, exp) 383 383 + t.Fatal("diffs not equal") 384 384 + } 385 385 + } 386 386 + 387 387 + func diffDiff(a, b []*DiffOp) { 388 388 + var i, j int 389 389 + 390 390 + for i < len(a) || j < len(b) { 391 391 + if i >= len(a) { 392 392 + fmt.Println("+: ", b[j]) 393 393 + j++ 394 394 + continue 395 395 + } 396 396 + 397 397 + if j >= len(b) { 398 398 + fmt.Println("-: ", a[i]) 399 399 + i++ 400 400 + continue 401 401 + } 402 402 + 403 403 + aa := a[i] 404 404 + bb := b[j] 405 405 + 406 406 + if diffOpEq(aa, bb) { 407 407 + fmt.Println("eq: ", i, j, aa.Tid) 408 408 + i++ 409 409 + j++ 410 410 + continue 411 411 + } 412 412 + 413 413 + if aa.Tid == bb.Tid { 414 414 + fmt.Println("~: ", aa, bb) 415 415 + i++ 416 416 + j++ 417 417 + continue 418 418 + } 419 419 + 420 420 + if aa.Tid < bb.Tid { 421 421 + fmt.Println("-: ", aa) 422 422 + i++ 423 423 + continue 424 424 + } else { 425 425 + fmt.Println("+: ", bb) 426 426 + j++ 427 427 + continue 428 428 + } 429 429 + } 430 430 + } 431 431 + 432 432 + func compareDiffs(a, b []*DiffOp) bool { 433 433 + if len(a) != len(b) { 434 434 + return false 435 435 + } 436 436 + 437 437 + for i := 0; i < len(a); i++ { 438 438 + aa := a[i] 439 439 + bb := b[i] 440 440 + 441 441 + if aa.Op != bb.Op || aa.Tid != bb.Tid || aa.NewCid != bb.NewCid || aa.OldCid != bb.OldCid { 442 442 + return false 443 443 + } 444 444 + } 445 445 + 446 446 + return true 447 447 + } 448 448 + 449 449 + func diffOpEq(aa, bb *DiffOp) bool { 450 450 + if aa.Op != bb.Op || aa.Tid != bb.Tid || aa.NewCid != bb.NewCid || aa.OldCid != bb.OldCid { 451 451 + return false 452 452 + } 453 453 + return true 454 454 + }

+1 -1

repo/repo.go

reviewed

··· 193 193 func (r *Repo) ForEach(ctx context.Context, prefix string, cb func(k string, v cid.Cid) error) error { 194 194 var com Commit 195 195 if err := r.cst.Get(ctx, r.sr.Root, &com); err != nil { 196 196 - return err 196 196 + return fmt.Errorf("failed to load commit: %w", err) 197 197 } 198 198 199 199 t := mst.LoadMST(r.cst, 32, com.Data)