this repo has no description
superficial refactoring
+51
-55
+8
-8
blockstore.py
+8
-8
blockstore.py
···
28
28
pass
29
29
30
30
@abstractmethod
31
-
def delete_block(self, key: bytes) -> None:
31
+
def del_block(self, key: bytes) -> None:
32
32
pass
33
33
34
34
···
55
55
raise KeyError("no block matches this key")
56
56
return value
57
57
58
-
def delete_block(self, key: bytes) -> None:
58
+
def del_block(self, key: bytes) -> None:
59
59
if key in self._state:
60
60
del self._state[key]
61
61
···
86
86
raise KeyError("no block matches this key")
87
87
return row[0]
88
88
89
-
def delete_block(self, key: bytes) -> None:
89
+
def del_block(self, key: bytes) -> None:
90
90
self._cur.execute(f"DELETE FROM {self.table} WHERE block_key=?", (key,))
91
91
92
92
···
109
109
except KeyError:
110
110
return self.lower.get_block(key)
111
111
112
-
def delete_block(self, key: bytes) -> None:
113
-
self.upper.delete_block(key)
112
+
def del_block(self, key: bytes) -> None:
113
+
self.upper.del_block(key)
114
114
115
115
116
116
···
130
130
131
131
print("hello ->", bs.get_block(b"hello"))
132
132
133
-
bs.delete_block(b"nothing") # nop
133
+
bs.del_block(b"nothing") # nop
134
134
135
-
bs.delete_block(b"hello")
135
+
bs.del_block(b"hello")
136
136
137
137
try:
138
138
bs.get_block(b"hello")
···
149
149
with sqlite3.connect(TEST_DB) as db:
150
150
bs = SqliteBlockStore(db)
151
151
print("hello ->", bs.get_block(b"hello"))
152
-
bs.delete_block(b"hello")
152
+
bs.del_block(b"hello")
153
153
154
154
try:
155
155
with sqlite3.connect(TEST_DB) as db:
+1
-1
carfile.py
+1
-1
carfile.py
+37
-41
mst.py
+37
-41
mst.py
···
227
227
def __init__(self, ns: NodeStore) -> None:
228
228
self.ns = ns
229
229
230
-
def put(self, root_cid: CID, key: str, val: CID) -> CID:
230
+
def put_record(self, root_cid: CID, key: str, val: CID) -> CID:
231
231
root = self.ns.get_node(root_cid)
232
232
if root.is_empty(): # special case for empty tree
233
233
return self._put_here(root, key, val).cid
234
234
return self._put_recursive(root, key, val, MSTNode.key_height(key), root.height).cid
235
235
236
-
def delete(self, root_cid: CID, key: str) -> CID:
236
+
def del_record(self, root_cid: CID, key: str) -> CID:
237
237
root = self.ns.get_node(root_cid)
238
238
239
239
# Note: the seemingly redundant outer .get().cid is required to transform
···
514
514
for deleted_key in deleted_kv.keys() - created_kv.keys():
515
515
yield ("deleted", deleted_key, deleted_kv[deleted_key].encode("base32")) #XXX: encode() is just for debugging
516
516
517
+
def very_slow_mst_diff(ns, root_a: CID, root_b: CID):
518
+
"""
519
+
This should return the same result as mst_diff, but it gets there in a very slow
520
+
yet less error-prone way, so it's useful for testing.
521
+
"""
522
+
a_nodes = set(NodeWalker(ns, root_a).iter_node_cids())
523
+
b_nodes = set(NodeWalker(ns, root_b).iter_node_cids())
524
+
return b_nodes - a_nodes, a_nodes - b_nodes
525
+
517
526
EMPTY_NODE_CID = MSTNode.empty_root().cid
518
527
519
528
def mst_diff(ns: NodeStore, root_a: CID, root_b: CID) -> Tuple[Set[CID], Set[CID]]: # created_deleted
520
-
created, deleted = mst_diff_recursive(NodeWalker(ns, root_a), NodeWalker(ns, root_b))
529
+
created = set() # MST nodes in b but not in a
530
+
deleted = set() # MST nodes in a but not in b
531
+
mst_diff_recursive(created, deleted, NodeWalker(ns, root_a), NodeWalker(ns, root_b))
521
532
middle = created & deleted # my algorithm has occasional false-positives
522
533
#assert(not middle) # this fails
523
534
#print("middle", len(middle))
···
530
541
created.add(EMPTY_NODE_CID)
531
542
return created, deleted
532
543
533
-
def very_slow_mst_diff(ns, root_a: CID, root_b: CID):
534
-
"""
535
-
This should return the same result as mst_diff, but it gets there in a very slow
536
-
yet less error-prone way, so it's useful for testing.
537
-
"""
538
-
a_nodes = set(NodeWalker(ns, root_a).iter_node_cids())
539
-
b_nodes = set(NodeWalker(ns, root_b).iter_node_cids())
540
-
return b_nodes - a_nodes, a_nodes - b_nodes
541
-
542
-
def mst_diff_recursive(a: NodeWalker, b: NodeWalker) -> Tuple[Set[CID], Set[CID]]: # created, deleted
543
-
mst_created = set() # MST nodes in b but not in a
544
-
mst_deleted = set() # MST nodes in a but not in b
545
-
544
+
def mst_diff_recursive(created: Set[CID], deleted: Set[CID], a: NodeWalker, b: NodeWalker): # created, deleted
546
545
# the easiest of all cases
547
546
if a.frame.node.cid == b.frame.node.cid:
548
-
return mst_created, mst_deleted # no difference
547
+
return # no difference
549
548
550
549
# trivial
551
550
if a.frame.node.is_empty():
552
-
mst_created |= set(b.iter_node_cids())
553
-
return mst_created, mst_deleted
551
+
#mst_deleted.add(a.frame.node.cid) # this doesn't work because it might've been a null subtree node
552
+
created |= set(b.iter_node_cids())
553
+
return
554
554
555
555
# likewise
556
556
if b.frame.node.is_empty():
557
-
mst_deleted |= set(a.iter_node_cids())
558
-
return mst_created, mst_deleted
557
+
#mst_created.add(b.frame.node.cid)
558
+
deleted |= set(a.iter_node_cids())
559
+
return
559
560
560
561
# now we're onto the hard part
561
562
···
571
572
"""
572
573
573
574
# NB: these will end up as false-positives if one tree is a subtree of the other
574
-
mst_created.add(b.frame.node.cid)
575
-
mst_deleted.add(a.frame.node.cid)
575
+
created.add(b.frame.node.cid)
576
+
deleted.add(a.frame.node.cid)
576
577
577
578
while True:
578
579
while a.rkey != b.rkey: # we need a loop because they might "leapfrog" each other
···
580
581
while a.rkey < b.rkey and not a.is_final:
581
582
if a.subtree: # recurse down every subtree
582
583
a.down()
583
-
mst_deleted.add(a.frame.node.cid)
584
+
deleted.add(a.frame.node.cid)
584
585
else:
585
586
a.right()
586
587
···
588
589
while b.rkey < a.rkey and not b.is_final:
589
590
if b.subtree: # recurse down every subtree
590
591
b.down()
591
-
mst_created.add(b.frame.node.cid)
592
+
created.add(b.frame.node.cid)
592
593
else:
593
594
b.right()
594
595
595
-
#print(a.rkey, a.stack[0].rkey, b.rkey, a.stack[0].rkey)
596
-
#assert(b.rkey == a.rkey)
597
-
# the rkeys match, but the subrees below us might not
596
+
# the rkeys now match, but the subrees below us might not
598
597
599
-
c, d = mst_diff_recursive(a.subtree_walker(), b.subtree_walker())
600
-
mst_created |= c
601
-
mst_deleted |= d
598
+
mst_diff_recursive(created, deleted, a.subtree_walker(), b.subtree_walker())
602
599
603
600
# check if we can still go right XXX: do we need to care about the case where one can, but the other can't?
604
601
# To consider: maybe if I just step a, b will catch up automagically
···
607
604
608
605
a.right()
609
606
b.right()
610
-
611
-
return mst_created, mst_deleted
607
+
612
608
613
609
if __name__ == "__main__":
614
610
if 0:
···
625
621
#enumerate_mst(ns, mst_root)
626
622
enumerate_mst_range(ns, mst_root, "app.bsky.feed.generator/", "app.bsky.feed.generator/\xff")
627
623
628
-
root2 = wrangler.delete(mst_root, "app.bsky.feed.generator/alttext")
629
-
root2 = wrangler.delete(root2, "app.bsky.feed.like/3kas3fyvkti22")
630
-
root2 = wrangler.put(root2, "app.bsky.feed.like/3kc3brpic2z2p", hash_to_cid(b"blah"))
624
+
root2 = wrangler.del_record(mst_root, "app.bsky.feed.generator/alttext")
625
+
root2 = wrangler.del_record(root2, "app.bsky.feed.like/3kas3fyvkti22")
626
+
root2 = wrangler.put_record(root2, "app.bsky.feed.like/3kc3brpic2z2p", hash_to_cid(b"blah"))
631
627
632
628
c, d = mst_diff(ns, mst_root, root2)
633
629
print("CREATED:")
···
649
645
wrangler = NodeWrangler(ns)
650
646
root = ns.get_node(None).cid
651
647
print(ns.pretty(root))
652
-
root = wrangler.put(root, "hello", hash_to_cid(b"blah"))
648
+
root = wrangler.put_record(root, "hello", hash_to_cid(b"blah"))
653
649
print(ns.pretty(root))
654
-
root = wrangler.put(root, "foo", hash_to_cid(b"bar"))
650
+
root = wrangler.put_record(root, "foo", hash_to_cid(b"bar"))
655
651
print(ns.pretty(root))
656
652
root_a = root
657
-
root = wrangler.put(root, "bar", hash_to_cid(b"bat"))
658
-
root = wrangler.put(root, "xyzz", hash_to_cid(b"bat"))
659
-
root = wrangler.delete(root, "foo")
653
+
root = wrangler.put_record(root, "bar", hash_to_cid(b"bat"))
654
+
root = wrangler.put_record(root, "xyzz", hash_to_cid(b"bat"))
655
+
root = wrangler.del_record(root, "foo")
660
656
print("=============")
661
657
print(ns.pretty(root_a))
662
658
print("=============")
+5
-5
mst_test.py
+5
-5
mst_test.py
···
13
13
for _ in range(10240 if PERF_BENCH else random.randrange(0, 32)):
14
14
k = random.randbytes(8).hex()
15
15
keys.append(k)
16
-
root = nw.put(root, k, hash_to_cid(random.randbytes(8)))
16
+
root = nw.put_record(root, k, hash_to_cid(random.randbytes(8)))
17
17
root_a = root
18
18
for _ in range(8 if PERF_BENCH else random.randrange(0, 8)):
19
19
# some random additions
20
-
root = nw.put(root, random.randbytes(8).hex(), hash_to_cid(random.randbytes(8)))
20
+
root = nw.put_record(root, random.randbytes(8).hex(), hash_to_cid(random.randbytes(8)))
21
21
if keys:
22
22
# some random modifications
23
23
for _ in range(4 if PERF_BENCH else random.randrange(0, 4)):
24
24
for k in random.choice(keys):
25
-
root = nw.put(root, k, hash_to_cid(random.randbytes(8)))
25
+
root = nw.put_record(root, k, hash_to_cid(random.randbytes(8)))
26
26
# some random deletions
27
27
for _ in range(4 if PERF_BENCH else random.randrange(0, 4)):
28
28
for k in random.choice(keys):
29
-
root = nw.delete(root, k)
29
+
root = nw.del_record(root, k)
30
30
31
31
diff_start = time.time()
32
32
c, d = mst_diff(ns, root_a, root)
···
42
42
43
43
if __name__ == "__main__":
44
44
duration = 0
45
-
for _ in range(1 if PERF_BENCH else 200):
45
+
for _ in range(1 if PERF_BENCH else 20000):
46
46
duration += random_test()
47
47
print("time spent diffing (ms):", duration*1000)