this repo has no description
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

drop dag-cbor and multiformats dependencies ๐ŸŽ‰

+47 -29
+1 -2
pyproject.toml
··· 19 19 "Operating System :: OS Independent", 20 20 ] 21 21 dependencies = [ 22 - "dag-cbor", 23 - "multiformats", 22 + "cbrrr@git+https://github.com/DavidBuchanan314/dag-cbrrr", 24 23 "more-itertools", 25 24 "lru-dict", 26 25 ]
+26 -5
src/atmst/blockstore/car_file.py
··· 1 1 from typing import Dict, List, Tuple, BinaryIO 2 2 import hashlib 3 3 4 - from multiformats import varint, CID 5 - import dag_cbor 4 + from cbrrr import parse_dag_cbor, CID 6 5 7 6 from . import BlockStore 8 7 8 + # should be equivalent to multiformats.varint.decode(), but not extremely slow for no reason. 9 + def parse_varint(stream: BinaryIO): 10 + n = 0 11 + shift = 0 12 + while True: 13 + val = stream.read(1) 14 + if not val: 15 + raise ValueError("eof") # match varint.decode() 16 + val = val[0] 17 + n |= (val & 0x7f) << shift 18 + if not val & 0x80: 19 + return n 20 + shift += 7 21 + 22 + def encode_varint(n: int) -> bytes: 23 + res = [] 24 + while n > 0x7f: 25 + res.append(0x80 | (n & 0x7f)) 26 + n >>= 7 27 + res.append(n) 28 + return bytes(res) 29 + 9 30 class ReadOnlyCARBlockStore(BlockStore): 10 31 """ 11 32 This is a sliiiightly unclean abstraction because BlockStores are indexed ··· 26 47 file.seek(0) 27 48 28 49 # parse out CAR header 29 - header_len = varint.decode(file) 50 + header_len = parse_varint(file) 30 51 header = file.read(header_len) 31 52 if len(header) != header_len: 32 53 raise EOFError("not enough CAR header bytes") 33 - header_obj = dag_cbor.decode(header) 54 + header_obj = parse_dag_cbor(header) 34 55 if header_obj.get("version") != 1: 35 56 raise ValueError(f"unsupported CAR version ({header_obj.get('version')})") 36 57 if len(header_obj["roots"]) != 1: ··· 41 62 self.block_offsets = {} 42 63 while True: 43 64 try: 44 - length = varint.decode(file) 65 + length = parse_varint(file) 45 66 except ValueError: 46 67 break # EOF 47 68 start = file.tell()
+8 -9
src/atmst/cartool.py
··· 4 4 import json 5 5 from typing import Tuple, Any, BinaryIO 6 6 7 - import dag_cbor 8 - from multiformats import CID, varint 7 + from cbrrr import encode_dag_cbor, parse_dag_cbor, CID 9 8 10 - from .blockstore.car_file import ReadOnlyCARBlockStore 9 + from .blockstore.car_file import ReadOnlyCARBlockStore, encode_varint 11 10 from .blockstore import OverlayBlockStore 12 11 from .mst.node_store import NodeStore 13 12 from .mst.node_walker import NodeWalker ··· 28 27 def open_car(car_path: str) -> Tuple[ReadOnlyCARBlockStore, dict]: 29 28 carfile = open(car_path, "rb") 30 29 bs = ReadOnlyCARBlockStore(carfile) 31 - commit = dag_cbor.decode(bs.get_block(bytes(bs.car_root))) 30 + commit = parse_dag_cbor(bs.get_block(bytes(bs.car_root))) 32 31 return bs, commit 33 32 34 33 def print_info(car_path: str) -> None: ··· 52 51 bs, commit = open_car(car_path) 53 52 for k, v in NodeWalker(NodeStore(bs), commit["data"]).iter_kv(): 54 53 if to_json: 55 - record = dag_cbor.decode(bs.get_block(bytes(v))) 54 + record = parse_dag_cbor(bs.get_block(bytes(v))) 56 55 print(f"{json.dumps(k)} -> {prettify_record(record)}") 57 56 else: 58 57 print(f"{json.dumps(k)} -> {v.encode('base32')}") ··· 70 69 if val is None: 71 70 print("Record not found!", file=sys.stderr) 72 71 sys.exit(-1) 73 - record = dag_cbor.decode(bs.get_block(bytes(val))) 72 + record = parse_dag_cbor(bs.get_block(bytes(val))) 74 73 print(prettify_record(record)) 75 74 76 75 def write_block(file: BinaryIO, data: bytes) -> None: 77 - file.write(varint.encode(len(data))) 76 + file.write(encode_varint(len(data))) 78 77 file.write(data) 79 78 80 79 def compact(car_in: str, car_out: str): 81 80 bs, commit = open_car(car_in) 82 81 with open(car_out, "wb") as carfile_out: 83 - new_header = dag_cbor.encode({ 82 + new_header = encode_dag_cbor({ 84 83 "version": 1, 85 84 "roots": [bs.car_root] 86 85 }) 87 86 write_block(carfile_out, new_header) 88 - write_block(carfile_out, bytes(bs.car_root) + dag_cbor.encode(commit)) 87 + write_block(carfile_out, bytes(bs.car_root) + encode_dag_cbor(commit)) 89 88 dedup = {bs.car_root} 90 89 91 90 for node in NodeWalker(NodeStore(bs), commit["data"]).iter_nodes():
+1 -1
src/atmst/mst/diff.py
··· 5 5 from functools import reduce 6 6 import json 7 7 8 - from multiformats import CID 8 + from cbrrr import CID 9 9 10 10 from .node import MSTNode 11 11 from .node_store import NodeStore
+4 -4
src/atmst/mst/node.py
··· 1 1 import hashlib 2 - import dag_cbor 3 2 import operator 4 - from multiformats import multihash, CID 5 3 from functools import cached_property 6 4 from more_itertools import ilen 7 5 from itertools import takewhile 8 6 from dataclasses import dataclass 9 7 from typing import Tuple, Self, Optional 8 + 9 + from cbrrr import encode_dag_cbor, parse_dag_cbor, CID 10 10 11 11 from ..util import hash_to_cid 12 12 ··· 73 73 "v": value, 74 74 }) 75 75 prev_key = key_bytes 76 - return dag_cbor.encode({ 76 + return encode_dag_cbor({ 77 77 "e": e, 78 78 "l": self.subtrees[0] 79 79 }) 80 80 81 81 @classmethod 82 82 def deserialise(cls, data: bytes) -> Self: 83 - cbor = dag_cbor.decode(data) 83 + cbor = parse_dag_cbor(data) 84 84 if len(cbor) != 2: # e, l 85 85 raise ValueError("malformed MST node") 86 86 subtrees = [cbor["l"]]
+1 -1
src/atmst/mst/node_store.py
··· 1 1 from typing import Optional, Dict 2 2 from functools import lru_cache 3 3 4 - from multiformats import CID 4 + from cbrrr import CID 5 5 from lru import LRU 6 6 7 7 from ..blockstore import BlockStore
+1 -1
src/atmst/mst/node_walker.py
··· 1 1 from dataclasses import dataclass 2 2 from typing import Tuple, Self, Optional, List, Iterable 3 3 4 - from multiformats import CID 4 + from cbrrr import CID 5 5 6 6 from .node import MSTNode 7 7 from .node_store import NodeStore
+1 -1
src/atmst/mst/node_wrangler.py
··· 1 1 from typing import Tuple, Optional, Any 2 2 3 - from multiformats import CID 3 + from cbrrr import CID 4 4 5 5 from .node import MSTNode 6 6 from .node_store import NodeStore
+4 -5
src/atmst/util.py
··· 1 - #import hashlib 2 - from multiformats import multihash, CID 1 + import hashlib 2 + from cbrrr import CID 3 3 4 4 def indent(msg: str) -> str: 5 5 ISTR = " " 6 6 return ISTR + msg.replace("\n", "\n"+ISTR) 7 7 8 8 def hash_to_cid(data: bytes, codec="dag-cbor") -> CID: 9 - #digest = b"\x12\x20" + hashlib.sha256(data).digest() 10 - digest = multihash.digest(data, "sha2-256") 11 - return CID("base32", 1, codec, digest) 9 + digest = b"\x12\x20" + hashlib.sha256(data).digest() 10 + return CID((b"\x00\x01q" if codec == "dag-cbor" else b'\x00\x01U') + digest) # XXX: this is a hack!