a collection of lightweight TypeScript packages for AT Protocol, the protocol powering Bluesky
atproto bluesky typescript npm
101
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat(car): initial commit

Mary 2e5fdff5 67cc493f

+421
+3
README.md
··· 43 43 <td><code>tid</code>: create and parse TID identifiers</td> 44 44 </tr> 45 45 <tr> 46 + <td><code>car</code>: read AT Protocol's CAR (content-addressable archive) repositories</td> 47 + </tr> 48 + <tr> 46 49 <td><code>cid</code>: create and parse the blessed CIDv1 format</td> 47 50 </tr> 48 51 <tr>
+9
packages/utilities/car/README.md
··· 1 + # @atcute/car 2 + 3 + read AT Protocol's CAR (content-addressable archive) repositories 4 + 5 + ```ts 6 + for (const { collection, rkey, record } of iterateAtpCar(buf)) { 7 + // ... 8 + } 9 + ```
+107
packages/utilities/car/lib/atproto-repo.ts
··· 1 + import * as CBOR from '@atcute/cbor'; 2 + import * as CID from '@atcute/cid'; 3 + 4 + import { fromUint8Array } from './reader.js'; 5 + 6 + const decoder = new TextDecoder(); 7 + 8 + export interface RepoEntry { 9 + collection: string; 10 + rkey: string; 11 + record: unknown; 12 + } 13 + 14 + export function* iterateAtpCar(buf: Uint8Array): Generator<RepoEntry> { 15 + const { roots, iterate } = fromUint8Array(new Uint8Array(buf)); 16 + assert(roots.length === 1, `expected only 1 root in the car archive; got=${roots.length}`); 17 + 18 + const blockmap: BlockMap = new Map(); 19 + for (const entry of iterate()) { 20 + blockmap.set(CID.format(entry.cid), entry.bytes); 21 + } 22 + 23 + const commit = readObject(blockmap, roots[0]) as Commit; 24 + for (const { key, cid } of walkEntries(blockmap, commit.data)) { 25 + const [collection, rkey] = key.split('/'); 26 + const record = readObject(blockmap, cid); 27 + 28 + yield { collection, rkey, record }; 29 + } 30 + } 31 + 32 + function readObject(map: BlockMap, link: CBOR.CIDLink): unknown { 33 + const cid = link.$link; 34 + 35 + const bytes = map.get(cid); 36 + assert(bytes != null, `cid not found in blockmap; cid=${cid}`); 37 + 38 + const data = CBOR.decode(bytes); 39 + 40 + return data; 41 + } 42 + 43 + function* walkEntries(map: BlockMap, pointer: CBOR.CIDLink): Generator<NodeEntry> { 44 + const data = readObject(map, pointer) as MstNode; 45 + const entries = data.e; 46 + 47 + let lastKey = ''; 48 + 49 + if (data.l !== null) { 50 + yield* walkEntries(map, data.l); 51 + } 52 + 53 + for (let i = 0, il = entries.length; i < il; i++) { 54 + const entry = entries[i]; 55 + 56 + const key_str = decoder.decode(CBOR.fromBytes(entry.k)); 57 + const key = lastKey.slice(0, entry.p) + key_str; 58 + 59 + lastKey = key; 60 + 61 + yield { key: key, cid: entry.v }; 62 + 63 + if (entry.t !== null) { 64 + yield* walkEntries(map, entry.t); 65 + } 66 + } 67 + } 68 + 69 + function assert(condition: boolean, message: string): asserts condition { 70 + if (!condition) { 71 + throw new Error(message); 72 + } 73 + } 74 + 75 + type BlockMap = Map<string, Uint8Array>; 76 + 77 + interface Commit { 78 + version: 3; 79 + did: string; 80 + data: CBOR.CIDLink; 81 + rev: string; 82 + prev: CBOR.CIDLink | null; 83 + sig: CBOR.Bytes; 84 + } 85 + 86 + interface TreeEntry { 87 + /** count of bytes shared with previous TreeEntry in this Node (if any) */ 88 + p: number; 89 + /** remainder of key for this TreeEntry, after "prefixlen" have been removed */ 90 + k: CBOR.Bytes; 91 + /** link to a sub-tree Node at a lower level which has keys sorting after this TreeEntry's key (to the "right"), but before the next TreeEntry's key in this Node (if any) */ 92 + v: CBOR.CIDLink; 93 + /** next subtree (to the right of leaf) */ 94 + t: CBOR.CIDLink | null; 95 + } 96 + 97 + interface MstNode { 98 + /** link to sub-tree Node on a lower level and with all keys sorting before keys at this node */ 99 + l: CBOR.CIDLink | null; 100 + /** ordered list of TreeEntry objects */ 101 + e: TreeEntry[]; 102 + } 103 + 104 + interface NodeEntry { 105 + key: string; 106 + cid: CBOR.CIDLink; 107 + }
+69
packages/utilities/car/lib/index.test.ts
··· 1 + import { expect, it } from 'bun:test'; 2 + 3 + import { iterateAtpCar } from './index.js'; 4 + 5 + it('decodes atproto car files', () => { 6 + const buf = Buffer.from( 7 + 'OqJlcm9vdHOB2CpYJQABcRIgkD8I0DL+GsJ3OKREpf9k73yHguuSEYzEiXPGueoJg8FndmVy' + 8 + 'c2lvbgGPAQFxEiDqG8o/D37K3hldhQTMRq9/Uvyf7X9evn9eB9ZdgpYq6qRlJHR5cGV2YXBw' + 9 + 'LmJza3kuYWN0b3IucHJvZmlsZWljcmVhdGVkQXR4GDIwMjQtMDItMjRUMTI6MTU6NDEuMjE5' + 10 + 'WmtkZXNjcmlwdGlvbm90ZXN0aW5nIGFjY291bnRrZGlzcGxheU5hbWVg4AEBcRIgkD8I0DL+' + 11 + 'GsJ3OKREpf9k73yHguuSEYzEiXPGueoJg8GmY2RpZHggZGlkOnBsYzpzcmNxb3UybTd1cXVv' + 12 + 'Z3lkNXhrNGI1eTVjcmV2bTNsNXE1ZmplbnRjMmRjc2lnWEDeWWEO5/vV6SmnbUrLRu9WhWqI' + 13 + 'kHKANGFOin3xqFc4fgtuYzkbFXFJDMQU06nBWxict8FQ8Kas9Mr2fDAh++vVZGRhdGHYKlgl' + 14 + 'AAFxEiB2ibkpj3r4cdTag9v2ipIe8fxyjUFOgCjZbtYnfhyJ2GRwcmV29md2ZXJzaW9uA6QB' + 15 + 'AXESIHaJuSmPevhx1NqD2/aKkh7x/HKNQU6AKNlu1id+HInYomFlgaRha1gbYXBwLmJza3ku' + 16 + 'YWN0b3IucHJvZmlsZS9zZWxmYXAAYXTYKlglAAFxEiBvSJJSaF/w/fee+UmoLV84FDwZRC7p' + 17 + 'pJX484MghY0rM2F22CpYJQABcRIg6hvKPw9+yt4ZXYUEzEavf1L8n+1/Xr5/XgfWXYKWKuph' + 18 + 'bPaBAQFxEiBvSJJSaF/w/fee+UmoLV84FDwZRC7ppJX484MghY0rM6JhZYGkYWtYIGFwcC5i' + 19 + 'c2t5LmZlZWQucG9zdC8za201eW1rNGhoazJ6YXAAYXT2YXbYKlglAAFxEiDj+gU903L3F3Ar' + 20 + 'WCg+aeQZYEiM3ooIxqHbVvbQPZvEbGFs9qECAXESIOP6BT3TcvcXcCtYKD5p5BlgSIzeigjG' + 21 + 'odtW9tA9m8RspWR0ZXh0dWJlZXAgYm9vcCBAbWFyeS5teS5pZGUkdHlwZXJhcHAuYnNreS5m' + 22 + 'ZWVkLnBvc3RlbGFuZ3OBYmVuZmZhY2V0c4GjZSR0eXBld2FwcC5ic2t5LnJpY2h0ZXh0LmZh' + 23 + 'Y2V0ZWluZGV4omdieXRlRW5kFWlieXRlU3RhcnQKaGZlYXR1cmVzgaJjZGlkeCBkaWQ6cGxj' + 24 + 'OmlhNzZrdm5uZGp1dGdlZGdneDJpYnJlbWUkdHlwZXgfYXBwLmJza3kucmljaHRleHQuZmFj' + 25 + 'ZXQjbWVudGlvbmljcmVhdGVkQXR4GDIwMjQtMDItMjRUMTI6MTY6MjAuNjM3Wg==', 26 + 'base64', 27 + ); 28 + 29 + // @ts-expect-error: node.js buffer it no likey 30 + const result = Array.from(iterateAtpCar(buf)); 31 + 32 + expect(result).toEqual([ 33 + { 34 + collection: 'app.bsky.actor.profile', 35 + rkey: 'self', 36 + record: { 37 + $type: 'app.bsky.actor.profile', 38 + createdAt: '2024-02-24T12:15:41.219Z', 39 + displayName: '', 40 + description: 'testing account', 41 + }, 42 + }, 43 + { 44 + collection: 'app.bsky.feed.post', 45 + rkey: '3km5ymk4hhk2z', 46 + record: { 47 + $type: 'app.bsky.feed.post', 48 + createdAt: '2024-02-24T12:16:20.637Z', 49 + langs: ['en'], 50 + text: 'beep boop @mary.my.id', 51 + facets: [ 52 + { 53 + $type: 'app.bsky.richtext.facet', 54 + index: { 55 + byteEnd: 21, 56 + byteStart: 10, 57 + }, 58 + features: [ 59 + { 60 + did: 'did:plc:ia76kvnndjutgedggx2ibrem', 61 + $type: 'app.bsky.richtext.facet#mention', 62 + }, 63 + ], 64 + }, 65 + ], 66 + }, 67 + }, 68 + ]); 69 + });
+2
packages/utilities/car/lib/index.ts
··· 1 + export { iterateAtpCar, type RepoEntry } from './atproto-repo.js'; 2 + export { fromUint8Array } from './reader.js';
+7
packages/utilities/car/lib/reader.ts
··· 1 + import { createUint8Reader } from './utilities/byte-reader.js'; 2 + import { createCarReader } from './utilities/sync-car-reader.js'; 3 + 4 + export const fromUint8Array = (buffer: Uint8Array) => { 5 + const reader = createUint8Reader(buffer); 6 + return createCarReader(reader); 7 + };
+35
packages/utilities/car/lib/utilities/byte-reader.ts
··· 1 + export interface SyncByteReader { 2 + readonly pos: number; 3 + upto(size: number): Uint8Array; 4 + exactly(size: number, seek: boolean): Uint8Array; 5 + seek(size: number): void; 6 + } 7 + 8 + export const createUint8Reader = (buf: Uint8Array): SyncByteReader => { 9 + let pos = 0; 10 + 11 + return { 12 + get pos() { 13 + return pos; 14 + }, 15 + 16 + seek(size) { 17 + pos += size; 18 + }, 19 + upto(size) { 20 + return buf.subarray(pos, pos + Math.min(size, buf.length - pos)); 21 + }, 22 + exactly(size, seek) { 23 + if (size > buf.length - pos) { 24 + throw new RangeError('unexpected end of data'); 25 + } 26 + 27 + const slice = buf.subarray(pos, pos + size); 28 + if (seek) { 29 + pos += size; 30 + } 31 + 32 + return slice; 33 + }, 34 + }; 35 + };
+116
packages/utilities/car/lib/utilities/sync-car-reader.ts
··· 1 + import * as CBOR from '@atcute/cbor'; 2 + import * as varint from '@atcute/varint'; 3 + import * as CID from '@atcute/cid'; 4 + 5 + import type { SyncByteReader } from './byte-reader.js'; 6 + 7 + interface CarV1Header { 8 + version: 1; 9 + roots: CBOR.CIDLink[]; 10 + } 11 + 12 + const isCarV1Header = (value: unknown): value is CarV1Header => { 13 + if (value === null || typeof value !== 'object') { 14 + return false; 15 + } 16 + 17 + const { version, roots } = value as CarV1Header; 18 + return version === 1 && Array.isArray(roots) && roots.every((root) => root instanceof CBOR.CIDLinkWrapper); 19 + }; 20 + 21 + const readVarint = (reader: SyncByteReader, size: number): number => { 22 + const buf = reader.upto(size); 23 + if (buf.length === 0) { 24 + throw new RangeError(`unexpected end of data`); 25 + } 26 + 27 + const [int, read] = varint.decode(buf); 28 + reader.seek(read); 29 + 30 + return int; 31 + }; 32 + 33 + const readHeader = (reader: SyncByteReader): CarV1Header => { 34 + const length = readVarint(reader, 8); 35 + if (length === 0) { 36 + throw new RangeError(`invalid car header; length=0`); 37 + } 38 + 39 + const rawHeader = reader.exactly(length, true); 40 + const header = CBOR.decode(rawHeader); 41 + if (!isCarV1Header(header)) { 42 + throw new TypeError(`expected a car v1 archive`); 43 + } 44 + 45 + return header; 46 + }; 47 + 48 + const readMultihashDigest = (reader: SyncByteReader): CID.Digest => { 49 + const first = reader.upto(8); 50 + 51 + const [code, codeOffset] = varint.decode(first); 52 + const [size, sizeOffset] = varint.decode(first.subarray(codeOffset)); 53 + 54 + const offset = codeOffset + sizeOffset; 55 + 56 + const bytes = reader.exactly(offset + size, true); 57 + const digest = bytes.subarray(offset); 58 + 59 + return { 60 + code: code, 61 + size: size, 62 + digest: digest, 63 + bytes: bytes, 64 + }; 65 + }; 66 + 67 + const readCid = (reader: SyncByteReader): CID.CID => { 68 + const version = readVarint(reader, 8); 69 + if (version !== 1) { 70 + throw new Error(`expected a cidv1`); 71 + } 72 + 73 + const codec = readVarint(reader, 8); 74 + const digest = readMultihashDigest(reader); 75 + 76 + const cid: CID.CID = { 77 + version: version, 78 + code: codec, 79 + digest: digest, 80 + bytes: CID.encode(version, codec, digest.bytes), 81 + }; 82 + 83 + return cid; 84 + }; 85 + 86 + const readBlockHeader = (reader: SyncByteReader): { cid: CID.CID; blockSize: number } => { 87 + const start = reader.pos; 88 + 89 + let size = readVarint(reader, 8); 90 + if (size === 0) { 91 + throw new Error(`invalid car section; length=0`); 92 + } 93 + 94 + size += reader.pos - start; 95 + 96 + const cid = readCid(reader); 97 + const blockSize = size - Number(reader.pos - start); 98 + 99 + return { cid, blockSize }; 100 + }; 101 + 102 + export const createCarReader = (reader: SyncByteReader) => { 103 + const { roots } = readHeader(reader); 104 + 105 + return { 106 + roots, 107 + *iterate(): Generator<{ cid: CID.CID; bytes: Uint8Array }> { 108 + while (reader.upto(8).length > 0) { 109 + const { cid, blockSize } = readBlockHeader(reader); 110 + const bytes = reader.exactly(blockSize, true); 111 + 112 + yield { cid, bytes }; 113 + } 114 + }, 115 + }; 116 + };
+30
packages/utilities/car/package.json
··· 1 + { 2 + "type": "module", 3 + "name": "@atcute/car", 4 + "version": "1.0.0", 5 + "description": "read AT Protocol's CAR (content-addressable archive) repositories", 6 + "license": "MIT", 7 + "repository": { 8 + "url": "https://codeberg.org/mary-ext/atcute" 9 + }, 10 + "files": [ 11 + "dist/" 12 + ], 13 + "exports": { 14 + ".": "./dist/index.js" 15 + }, 16 + "sideEffects": false, 17 + "scripts": { 18 + "build": "tsc --project tsconfig.build.json", 19 + "test": "bun test --coverage", 20 + "prepublish": "rm -rf dist; pnpm run build" 21 + }, 22 + "devDependencies": { 23 + "@types/bun": "^1.1.10" 24 + }, 25 + "dependencies": { 26 + "@atcute/cbor": "workspace:^", 27 + "@atcute/cid": "workspace:^", 28 + "@atcute/varint": "workspace:^" 29 + } 30 + }
+4
packages/utilities/car/tsconfig.build.json
··· 1 + { 2 + "extends": "./tsconfig.json", 3 + "exclude": ["**/*.test.ts", "**/*.bench.ts"] 4 + }
+23
packages/utilities/car/tsconfig.json
··· 1 + { 2 + "compilerOptions": { 3 + "types": ["bun"], 4 + "outDir": "dist/", 5 + "esModuleInterop": true, 6 + "skipLibCheck": true, 7 + "target": "ESNext", 8 + "allowJs": true, 9 + "resolveJsonModule": true, 10 + "moduleDetection": "force", 11 + "isolatedModules": true, 12 + "verbatimModuleSyntax": true, 13 + "strict": true, 14 + "noImplicitOverride": true, 15 + "noUnusedLocals": true, 16 + "noUnusedParameters": true, 17 + "noFallthroughCasesInSwitch": true, 18 + "module": "NodeNext", 19 + "sourceMap": true, 20 + "declaration": true, 21 + }, 22 + "include": ["lib"], 23 + }
+16
pnpm-lock.yaml
··· 161 161 specifier: ^1.1.10 162 162 version: 1.1.10 163 163 164 + packages/utilities/car: 165 + dependencies: 166 + '@atcute/cbor': 167 + specifier: workspace:^ 168 + version: link:../cbor 169 + '@atcute/cid': 170 + specifier: workspace:^ 171 + version: link:../cid 172 + '@atcute/varint': 173 + specifier: workspace:^ 174 + version: link:../varint 175 + devDependencies: 176 + '@types/bun': 177 + specifier: ^1.1.10 178 + version: 1.1.10 179 + 164 180 packages/utilities/cbor: 165 181 dependencies: 166 182 '@atcute/base32':