this repo has no description
drop dag-cbor and multiformats dependencies 🎉
+47
-29
+1
-2
pyproject.toml
+1
-2
pyproject.toml
+26
-5
src/atmst/blockstore/car_file.py
+26
-5
src/atmst/blockstore/car_file.py
···
1
1
from typing import Dict, List, Tuple, BinaryIO
2
2
import hashlib
3
3
4
-
from multiformats import varint, CID
5
-
import dag_cbor
4
+
from cbrrr import parse_dag_cbor, CID
6
5
7
6
from . import BlockStore
8
7
8
+
# should be equivalent to multiformats.varint.decode(), but not extremely slow for no reason.
9
+
def parse_varint(stream: BinaryIO):
10
+
n = 0
11
+
shift = 0
12
+
while True:
13
+
val = stream.read(1)
14
+
if not val:
15
+
raise ValueError("eof") # match varint.decode()
16
+
val = val[0]
17
+
n |= (val & 0x7f) << shift
18
+
if not val & 0x80:
19
+
return n
20
+
shift += 7
21
+
22
+
def encode_varint(n: int) -> bytes:
23
+
res = []
24
+
while n > 0x7f:
25
+
res.append(0x80 | (n & 0x7f))
26
+
n >>= 7
27
+
res.append(n)
28
+
return bytes(res)
29
+
9
30
class ReadOnlyCARBlockStore(BlockStore):
10
31
"""
11
32
This is a sliiiightly unclean abstraction because BlockStores are indexed
···
26
47
file.seek(0)
27
48
28
49
# parse out CAR header
29
-
header_len = varint.decode(file)
50
+
header_len = parse_varint(file)
30
51
header = file.read(header_len)
31
52
if len(header) != header_len:
32
53
raise EOFError("not enough CAR header bytes")
33
-
header_obj = dag_cbor.decode(header)
54
+
header_obj = parse_dag_cbor(header)
34
55
if header_obj.get("version") != 1:
35
56
raise ValueError(f"unsupported CAR version ({header_obj.get('version')})")
36
57
if len(header_obj["roots"]) != 1:
···
41
62
self.block_offsets = {}
42
63
while True:
43
64
try:
44
-
length = varint.decode(file)
65
+
length = parse_varint(file)
45
66
except ValueError:
46
67
break # EOF
47
68
start = file.tell()
+8
-9
src/atmst/cartool.py
+8
-9
src/atmst/cartool.py
···
4
4
import json
5
5
from typing import Tuple, Any, BinaryIO
6
6
7
-
import dag_cbor
8
-
from multiformats import CID, varint
7
+
from cbrrr import encode_dag_cbor, parse_dag_cbor, CID
9
8
10
-
from .blockstore.car_file import ReadOnlyCARBlockStore
9
+
from .blockstore.car_file import ReadOnlyCARBlockStore, encode_varint
11
10
from .blockstore import OverlayBlockStore
12
11
from .mst.node_store import NodeStore
13
12
from .mst.node_walker import NodeWalker
···
28
27
def open_car(car_path: str) -> Tuple[ReadOnlyCARBlockStore, dict]:
29
28
carfile = open(car_path, "rb")
30
29
bs = ReadOnlyCARBlockStore(carfile)
31
-
commit = dag_cbor.decode(bs.get_block(bytes(bs.car_root)))
30
+
commit = parse_dag_cbor(bs.get_block(bytes(bs.car_root)))
32
31
return bs, commit
33
32
34
33
def print_info(car_path: str) -> None:
···
52
51
bs, commit = open_car(car_path)
53
52
for k, v in NodeWalker(NodeStore(bs), commit["data"]).iter_kv():
54
53
if to_json:
55
-
record = dag_cbor.decode(bs.get_block(bytes(v)))
54
+
record = parse_dag_cbor(bs.get_block(bytes(v)))
56
55
print(f"{json.dumps(k)} -> {prettify_record(record)}")
57
56
else:
58
57
print(f"{json.dumps(k)} -> {v.encode('base32')}")
···
70
69
if val is None:
71
70
print("Record not found!", file=sys.stderr)
72
71
sys.exit(-1)
73
-
record = dag_cbor.decode(bs.get_block(bytes(val)))
72
+
record = parse_dag_cbor(bs.get_block(bytes(val)))
74
73
print(prettify_record(record))
75
74
76
75
def write_block(file: BinaryIO, data: bytes) -> None:
77
-
file.write(varint.encode(len(data)))
76
+
file.write(encode_varint(len(data)))
78
77
file.write(data)
79
78
80
79
def compact(car_in: str, car_out: str):
81
80
bs, commit = open_car(car_in)
82
81
with open(car_out, "wb") as carfile_out:
83
-
new_header = dag_cbor.encode({
82
+
new_header = encode_dag_cbor({
84
83
"version": 1,
85
84
"roots": [bs.car_root]
86
85
})
87
86
write_block(carfile_out, new_header)
88
-
write_block(carfile_out, bytes(bs.car_root) + dag_cbor.encode(commit))
87
+
write_block(carfile_out, bytes(bs.car_root) + encode_dag_cbor(commit))
89
88
dedup = {bs.car_root}
90
89
91
90
for node in NodeWalker(NodeStore(bs), commit["data"]).iter_nodes():
+1
-1
src/atmst/mst/diff.py
+1
-1
src/atmst/mst/diff.py
+4
-4
src/atmst/mst/node.py
+4
-4
src/atmst/mst/node.py
···
1
1
import hashlib
2
-
import dag_cbor
3
2
import operator
4
-
from multiformats import multihash, CID
5
3
from functools import cached_property
6
4
from more_itertools import ilen
7
5
from itertools import takewhile
8
6
from dataclasses import dataclass
9
7
from typing import Tuple, Self, Optional
8
+
9
+
from cbrrr import encode_dag_cbor, parse_dag_cbor, CID
10
10
11
11
from ..util import hash_to_cid
12
12
···
73
73
"v": value,
74
74
})
75
75
prev_key = key_bytes
76
-
return dag_cbor.encode({
76
+
return encode_dag_cbor({
77
77
"e": e,
78
78
"l": self.subtrees[0]
79
79
})
80
80
81
81
@classmethod
82
82
def deserialise(cls, data: bytes) -> Self:
83
-
cbor = dag_cbor.decode(data)
83
+
cbor = parse_dag_cbor(data)
84
84
if len(cbor) != 2: # e, l
85
85
raise ValueError("malformed MST node")
86
86
subtrees = [cbor["l"]]
+1
-1
src/atmst/mst/node_store.py
+1
-1
src/atmst/mst/node_store.py
+1
-1
src/atmst/mst/node_walker.py
+1
-1
src/atmst/mst/node_walker.py
+1
-1
src/atmst/mst/node_wrangler.py
+1
-1
src/atmst/mst/node_wrangler.py
+4
-5
src/atmst/util.py
+4
-5
src/atmst/util.py
···
1
-
#import hashlib
2
-
from multiformats import multihash, CID
1
+
import hashlib
2
+
from cbrrr import CID
3
3
4
4
def indent(msg: str) -> str:
5
5
ISTR = " "
6
6
return ISTR + msg.replace("\n", "\n"+ISTR)
7
7
8
8
def hash_to_cid(data: bytes, codec="dag-cbor") -> CID:
9
-
#digest = b"\x12\x20" + hashlib.sha256(data).digest()
10
-
digest = multihash.digest(data, "sha2-256")
11
-
return CID("base32", 1, codec, digest)
9
+
digest = b"\x12\x20" + hashlib.sha256(data).digest()
10
+
return CID((b"\x00\x01q" if codec == "dag-cbor" else b'\x00\x01U') + digest) # XXX: this is a hack!