🌿 Collaborative wiki on ATProto
lichen.wiki
atproto
Move to use TID from atcute repo
+609
-169
+12
bun.lock
+12
bun.lock
···
5
5
"": {
6
6
"name": "atwiki",
7
7
"dependencies": {
8
+
"@atcute/cid": "^2.4.1",
9
+
"@atcute/tid": "^1.1.2",
8
10
"@atproto/api": "^0.13.0",
9
11
"@atproto/identity": "^0.4.12",
10
12
"@atproto/jwk-jose": "^0.1.0",
···
51
53
"@atproto/did/zod": "3.23.8",
52
54
},
53
55
"packages": {
56
+
"@atcute/cid": ["@atcute/cid@2.4.1", "", { "dependencies": { "@atcute/multibase": "^1.1.8", "@atcute/uint8array": "^1.1.1" } }, "sha512-bwhna69RCv7yetXudtj+2qrMPYvhhIQqvJz6YUpUS98v7OdF3X2dnye9Nig2NDrklZcuyOsu7sQo7GOykJXRLQ=="],
57
+
58
+
"@atcute/multibase": ["@atcute/multibase@1.2.0", "", { "dependencies": { "@atcute/uint8array": "^1.1.1" } }, "sha512-ZK2GRra+qIYq9nNuQB52m2ul0hOmCQEtPobGfTSUxm7pF0OGEkWGkWHugFhNEDVzHzTwPxHp6VGotdZFue4lYQ=="],
59
+
60
+
"@atcute/tid": ["@atcute/tid@1.1.2", "", { "dependencies": { "@atcute/time-ms": "^1.2.2" } }, "sha512-bmPuOX/TOfcm/vsK9vM98spjkcx2wgd9S2PeK5oLgEr8IbNRPq7iMCAPzOL1nu5XAW3LlkOYQEbYRcw5vcQ37w=="],
61
+
62
+
"@atcute/time-ms": ["@atcute/time-ms@1.3.2", "", {}, "sha512-F+qOyR9pO55g1d/QmN+Gr+fimoUQQLusdGSB6pjV0wW5KPILR4oQ4e2ZhWzqUbeHLAgWvgoTTMsMDdz62Xa2tg=="],
63
+
64
+
"@atcute/uint8array": ["@atcute/uint8array@1.1.1", "", {}, "sha512-3LsC8XB8TKe9q/5hOA5sFuzGaIFdJZJNewC5OKa3o/eU6+K7JR6see9Zy2JbQERNVnRl11EzbNov1efgLMAs4g=="],
65
+
54
66
"@atproto-labs/did-resolver": ["@atproto-labs/did-resolver@0.1.4", "", { "dependencies": { "@atproto-labs/fetch": "0.1.1", "@atproto-labs/pipe": "0.1.0", "@atproto-labs/simple-store": "0.1.1", "@atproto-labs/simple-store-memory": "0.1.1", "@atproto/did": "0.1.2", "zod": "^3.23.8" } }, "sha512-5d+LHScS2ueYsFRjMOC3c1EwM2ui1yBVbBA0yY3MH7aydbljm5D28scsOVuymIhHwPFwcGvZbMON4PVSfpBbbQ=="],
55
67
56
68
"@atproto-labs/fetch": ["@atproto-labs/fetch@0.1.1", "", { "dependencies": { "@atproto-labs/pipe": "0.1.0" }, "optionalDependencies": { "zod": "^3.23.8" } }, "sha512-X1zO1MDoJzEurbWXMAe1H8EZ995Xam/aXdxhGVrXmOMyPDuvBa1oxwh/kQNZRCKcMQUbiwkk+Jfq6ZkTuvGbww=="],
+1
-1
lexicons/wiki.lichen.noteRevision.json
+1
-1
lexicons/wiki.lichen.noteRevision.json
+2
package.json
+2
package.json
src/lib/blob.ts
src/lib/attachments.ts
src/lib/blob.ts
src/lib/attachments.ts
+3
-19
src/lib/import-export/export.ts
+3
-19
src/lib/import-export/export.ts
···
6
6
listNotesWithContent,
7
7
} from "../../server/db/queries/index.ts";
8
8
import { MIME_TO_EXT } from "../constants.ts";
9
-
import { resolvePdsEndpoint } from "../identity.ts";
9
+
import { fetchVerifiedBlob } from "../pds-fetch.ts";
10
10
import { rewriteForExport } from "./markdown-transform.ts";
11
11
12
12
const INVALID_FILENAME_CHARS = /[/\\:*?"<>|]/g;
···
63
63
64
64
for (const [cid, ref] of uniqueBlobs) {
65
65
try {
66
-
const data = await fetchBlobFromPds(ref.did, cid);
67
-
if (data) {
68
-
blobData.set(cid, data);
69
-
}
66
+
const { data } = await fetchVerifiedBlob(ref.did, cid);
67
+
blobData.set(cid, data);
70
68
} catch {
71
69
warnings.push(`Could not fetch blob ${cid}`);
72
70
}
···
99
97
}
100
98
101
99
return zipSync(zipEntries);
102
-
}
103
-
104
-
async function fetchBlobFromPds(
105
-
did: string,
106
-
cid: string,
107
-
): Promise<Uint8Array | null> {
108
-
const pdsEndpoint = await resolvePdsEndpoint(did);
109
-
if (!pdsEndpoint) return null;
110
-
111
-
const url = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`;
112
-
const response = await fetch(url);
113
-
if (!response.ok) return null;
114
-
115
-
return new Uint8Array(await response.arrayBuffer());
116
100
}
117
101
118
102
function sanitizeFilename(title: string): string {
+4
-4
src/lib/import-export/import.ts
+4
-4
src/lib/import-export/import.ts
···
1
+
import * as TID from "@atcute/tid";
1
2
import { writeNoteRecord, writeRevisionRecord } from "../../atproto/pds.ts";
2
3
import type { getAgent } from "../../atproto/session.ts";
3
4
import { createNote } from "../../server/db/queries/index.ts";
4
5
import type { RequestContext } from "../access.ts";
5
-
import { type BlobMeta, buildBlobsForContent } from "../blob.ts";
6
+
import { type BlobMeta, buildBlobsForContent } from "../attachments.ts";
6
7
import { createDiff } from "../diff.ts";
7
8
import type { Messages } from "../i18n/index.ts";
8
9
import { processImage } from "../image.ts";
9
10
import { withPdsError } from "../orchestrators/helpers.ts";
10
11
import { createWikiCore, type WikiFormFields } from "../orchestrators/wiki.ts";
11
-
import { generateTid } from "../tid.ts";
12
12
import { rewriteForImport } from "./markdown-transform.ts";
13
13
import type { ImportedImage } from "./types.ts";
14
14
import { parseImportZip } from "./zip-parse.ts";
···
54
54
const content = rewriteForImport(note.content, slugMap, imageMap);
55
55
const blobs = buildBlobsForContent(content, blobMeta);
56
56
57
-
const noteTid = generateTid();
58
-
const revisionTid = generateTid();
57
+
const noteTid = TID.now();
58
+
const revisionTid = TID.now();
59
59
const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`;
60
60
const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`;
61
61
+2
-2
src/lib/limits.ts
+2
-2
src/lib/limits.ts
+3
-3
src/lib/orchestrators/bookmark.ts
+3
-3
src/lib/orchestrators/bookmark.ts
···
1
+
import * as TID from "@atcute/tid";
1
2
import { deleteRecord, writeBookmarkRecord } from "../../atproto/pds.ts";
2
3
import { getAgent, type Session } from "../../atproto/session.ts";
3
4
import {
···
7
8
} from "../../server/db/queries/index.ts";
8
9
import { parseAtUri } from "../at-uri.ts";
9
10
import { COLLECTIONS } from "../constants.ts";
10
-
import { currentTimestamp, generateTid } from "../tid.ts";
11
11
import { withPdsError } from "./helpers.ts";
12
12
13
13
export async function addBookmarkAction(
···
15
15
wikiAtUri: string,
16
16
session: Session | null,
17
17
): Promise<void> {
18
-
const now = currentTimestamp();
19
-
const tid = generateTid();
18
+
const now = new Date().toISOString();
19
+
const tid = TID.now();
20
20
const atUri = `at://${did}/${COLLECTIONS.bookmark}/${tid}`;
21
21
22
22
if (session) {
+8
-8
src/lib/orchestrators/membership.ts
+8
-8
src/lib/orchestrators/membership.ts
···
1
+
import * as TID from "@atcute/tid";
1
2
import {
2
3
deleteRecord,
3
4
writeMemberRequestRecord,
···
18
19
import { COLLECTIONS } from "../constants.ts";
19
20
import { ForbiddenError, NotFoundError, ValidationError } from "../errors.ts";
20
21
import { t } from "../i18n/index.ts";
21
-
import { currentTimestamp, generateTid } from "../tid.ts";
22
22
import { withPdsError } from "./helpers.ts";
23
23
24
24
/**
···
33
33
}
34
34
35
35
const session = ctx.session;
36
-
const now = currentTimestamp();
37
-
const tid = generateTid();
36
+
const now = new Date().toISOString();
37
+
const tid = TID.now();
38
38
const atUri = `at://${session.did}/wiki.lichen.memberRequest/${tid}`;
39
39
40
40
await withPdsError("request access", async () => {
···
63
63
const did = ctx.did;
64
64
if (!did) throw new ForbiddenError();
65
65
66
-
const now = currentTimestamp();
67
-
const membershipTid = generateTid();
66
+
const now = new Date().toISOString();
67
+
const membershipTid = TID.now();
68
68
const membershipAtUri = `at://${did}/wiki.lichen.membership/${membershipTid}`;
69
69
70
70
if (ctx.session) {
···
111
111
const did = ctx.did;
112
112
if (!did) throw new ForbiddenError();
113
113
114
-
const newTid = generateTid();
114
+
const newTid = TID.now();
115
115
const newAtUri = `at://${did}/wiki.lichen.membership/${newTid}`;
116
116
117
117
if (ctx.session) {
···
168
168
const did = ctx.did;
169
169
if (!did) throw new ForbiddenError();
170
170
171
-
const now = currentTimestamp();
172
-
const tid = generateTid();
171
+
const now = new Date().toISOString();
172
+
const tid = TID.now();
173
173
const atUri = `at://${did}/wiki.lichen.membership/${tid}`;
174
174
175
175
if (ctx.session) {
+7
-7
src/lib/orchestrators/note.ts
+7
-7
src/lib/orchestrators/note.ts
···
1
+
import * as TID from "@atcute/tid";
1
2
import {
2
3
deleteRecord,
3
4
writeNoteRecord,
···
17
18
type BlobMeta,
18
19
buildBlobsForContent,
19
20
parseBlobMetadata,
20
-
} from "../blob.ts";
21
+
} from "../attachments.ts";
21
22
import { COLLECTIONS } from "../constants.ts";
22
23
import { createDiff } from "../diff.ts";
23
24
import { ForbiddenError, NotFoundError, ValidationError } from "../errors.ts";
24
25
import { fmt, type Messages } from "../i18n/index.ts";
25
26
import { LIMITS } from "../limits.ts";
26
27
import { validateNewNote } from "../note-validation.ts";
27
-
import { currentTimestamp, generateTid } from "../tid.ts";
28
28
import { withPdsError } from "./helpers.ts";
29
29
30
30
interface NoteFormFields {
···
83
83
blobs: ReturnType<typeof buildBlobsForContent>,
84
84
): Promise<void> {
85
85
const diff = createDiff(oldContent, newContent);
86
-
const now = currentTimestamp();
86
+
const now = new Date().toISOString();
87
87
await writeRevisionRecord(
88
88
agent,
89
89
did,
···
121
121
if (!did) throw new ForbiddenError();
122
122
123
123
// Generate TIDs once — shared between PDS and DB writes
124
-
const noteTid = generateTid();
125
-
const revisionTid = generateTid();
124
+
const noteTid = TID.now();
125
+
const revisionTid = TID.now();
126
126
const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`;
127
127
const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`;
128
128
129
129
if (ctx.session) {
130
130
const agent = await getAgent(ctx.session);
131
-
const now = currentTimestamp();
131
+
const now = new Date().toISOString();
132
132
await withPdsError("create note", async () => {
133
133
await writeNoteRecord(
134
134
agent,
···
202
202
const currentNote = getCurrentNote(ctx.wiki.slug, noteSlug);
203
203
204
204
// Generate revision TID once — shared between PDS and DB writes
205
-
const revisionTid = generateTid();
205
+
const revisionTid = TID.now();
206
206
const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`;
207
207
const currentContent = currentNote?.content ?? "";
208
208
+5
-5
src/lib/orchestrators/wiki.ts
+5
-5
src/lib/orchestrators/wiki.ts
···
1
+
import * as TID from "@atcute/tid";
1
2
import {
2
3
deleteRecord,
3
4
writeMembershipRecord,
···
22
23
import { fmt, type Messages, t } from "../i18n/index.ts";
23
24
import { LIMITS } from "../limits.ts";
24
25
import { isValidSlug, slugify } from "../slug.ts";
25
-
import { currentTimestamp, generateTid } from "../tid.ts";
26
26
import { withPdsError } from "./helpers.ts";
27
27
28
28
export interface WikiFormFields {
···
78
78
79
79
const validVisibility =
80
80
fields.visibility === "private" ? "private" : ("public" as const);
81
-
const now = currentTimestamp();
81
+
const now = new Date().toISOString();
82
82
const did = ctx.did;
83
83
if (!did) throw new ForbiddenError();
84
84
···
105
105
});
106
106
}
107
107
108
-
const membershipTid = generateTid();
108
+
const membershipTid = TID.now();
109
109
const membershipAtUri = `at://${did}/wiki.lichen.membership/${membershipTid}`;
110
110
111
111
if (agent) {
···
156
156
157
157
// Create home note — TIDs shared between PDS and DB writes
158
158
const homeContent = `# Welcome to ${fields.name}\n\nThis is the home page of your wiki. Edit it to get started.`;
159
-
const noteTid = generateTid();
160
-
const revisionTid = generateTid();
159
+
const noteTid = TID.now();
160
+
const revisionTid = TID.now();
161
161
const noteAtUri = `at://${did}/wiki.lichen.note/${noteTid}`;
162
162
const revisionAtUri = `at://${did}/wiki.lichen.noteRevision/${revisionTid}`;
163
163
+161
src/lib/pds-fetch.ts
+161
src/lib/pds-fetch.ts
···
1
+
import * as CID from "@atcute/cid";
2
+
import { AppError } from "./errors.ts";
3
+
import { resolvePdsEndpoint } from "./identity.ts";
4
+
import { LIMITS } from "./limits.ts";
5
+
6
+
type BlobFetchReason =
7
+
| "invalid-cid"
8
+
| "pds-resolve-failed"
9
+
| "pds-not-found"
10
+
| "non-https-pds"
11
+
| "fetch-failed"
12
+
| "timeout"
13
+
| "upstream-error"
14
+
| "too-large"
15
+
| "cid-mismatch";
16
+
17
+
export class BlobFetchError extends AppError {
18
+
readonly reason: BlobFetchReason;
19
+
constructor(reason: BlobFetchReason, status: number, message: string) {
20
+
super(message, status);
21
+
this.reason = reason;
22
+
}
23
+
}
24
+
25
+
interface VerifiedBlob {
26
+
data: Uint8Array;
27
+
mimeType: string;
28
+
}
29
+
30
+
/**
31
+
* Fetch a blob from its owning PDS and verify it.
32
+
*
33
+
* The PDS is resolved from the DID, then `com.atproto.sync.getBlob` is called
34
+
* with these guarantees:
35
+
*
36
+
* - SSRF guard: only HTTPS endpoints, except localhost for dev
37
+
* - Timeout: aborts after `LIMITS.blobProxy.timeoutMs`
38
+
* - Size cap: rejects on Content-Length, then enforces during streaming
39
+
* (the streaming check is the actual security boundary — a malicious PDS
40
+
* can lie about Content-Length)
41
+
* - CID verification: hashes the bytes and rejects if they don't match the
42
+
* requested CID (a malicious PDS could otherwise serve arbitrary content
43
+
* for any CID)
44
+
*/
45
+
export async function fetchVerifiedBlob(
46
+
did: string,
47
+
cid: string,
48
+
fetchFn: typeof fetch = fetch,
49
+
resolveEndpoint: (did: string) => Promise<string | null> = resolvePdsEndpoint,
50
+
): Promise<VerifiedBlob> {
51
+
let expected: CID.Cid;
52
+
try {
53
+
expected = CID.fromString(cid);
54
+
} catch {
55
+
throw new BlobFetchError("invalid-cid", 400, "Invalid CID");
56
+
}
57
+
58
+
let pdsEndpoint: string;
59
+
try {
60
+
const resolved = await resolveEndpoint(did);
61
+
if (!resolved) {
62
+
throw new BlobFetchError("pds-not-found", 404, "PDS not found for DID");
63
+
}
64
+
pdsEndpoint = resolved;
65
+
} catch (err) {
66
+
if (err instanceof BlobFetchError) throw err;
67
+
throw new BlobFetchError(
68
+
"pds-resolve-failed",
69
+
502,
70
+
"Failed to resolve DID",
71
+
);
72
+
}
73
+
74
+
const pdsUrl = new URL(pdsEndpoint);
75
+
const isLocal =
76
+
pdsUrl.hostname === "localhost" || pdsUrl.hostname === "127.0.0.1";
77
+
if (pdsUrl.protocol !== "https:" && !isLocal) {
78
+
throw new BlobFetchError(
79
+
"non-https-pds",
80
+
502,
81
+
"PDS endpoint must be HTTPS",
82
+
);
83
+
}
84
+
85
+
const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`;
86
+
87
+
let upstream: Response;
88
+
try {
89
+
upstream = await fetchFn(blobUrl, {
90
+
signal: AbortSignal.timeout(LIMITS.blobProxy.timeoutMs),
91
+
});
92
+
} catch (err) {
93
+
if (err instanceof Error && err.name === "TimeoutError") {
94
+
throw new BlobFetchError("timeout", 504, "PDS request timed out");
95
+
}
96
+
throw new BlobFetchError(
97
+
"fetch-failed",
98
+
502,
99
+
"Failed to fetch blob from PDS",
100
+
);
101
+
}
102
+
103
+
if (!upstream.ok) {
104
+
await upstream.body?.cancel().catch(() => {});
105
+
throw new BlobFetchError(
106
+
"upstream-error",
107
+
upstream.status,
108
+
"Blob not found",
109
+
);
110
+
}
111
+
112
+
const announced = upstream.headers.get("content-length");
113
+
if (announced !== null && Number(announced) > LIMITS.blobProxy.maxBytes) {
114
+
await upstream.body?.cancel().catch(() => {});
115
+
throw new BlobFetchError("too-large", 413, "Blob too large");
116
+
}
117
+
118
+
const mimeType =
119
+
upstream.headers.get("content-type") ?? "application/octet-stream";
120
+
121
+
if (upstream.body === null) {
122
+
throw new BlobFetchError("upstream-error", 502, "PDS returned empty body");
123
+
}
124
+
125
+
const reader = upstream.body.getReader();
126
+
const chunks: Uint8Array[] = [];
127
+
let total = 0;
128
+
try {
129
+
while (true) {
130
+
const { done, value } = await reader.read();
131
+
if (done) break;
132
+
total += value.byteLength;
133
+
if (total > LIMITS.blobProxy.maxBytes) {
134
+
await reader.cancel().catch(() => {});
135
+
throw new BlobFetchError("too-large", 413, "Blob too large");
136
+
}
137
+
chunks.push(value);
138
+
}
139
+
} catch (err) {
140
+
if (err instanceof BlobFetchError) throw err;
141
+
throw new BlobFetchError("fetch-failed", 502, "Failed to read blob body");
142
+
}
143
+
144
+
const data = new Uint8Array(total);
145
+
let offset = 0;
146
+
for (const chunk of chunks) {
147
+
data.set(chunk, offset);
148
+
offset += chunk.byteLength;
149
+
}
150
+
151
+
const computed = await CID.create(0x55, data);
152
+
if (!CID.equals(computed, expected)) {
153
+
throw new BlobFetchError(
154
+
"cid-mismatch",
155
+
502,
156
+
"Blob content does not match CID",
157
+
);
158
+
}
159
+
160
+
return { data, mimeType };
161
+
}
-23
src/lib/tid.ts
-23
src/lib/tid.ts
···
1
-
const BASE32_CHARS = "234567abcdefghijklmnopqrstuvwxyz";
2
-
3
-
export function currentTimestamp(): string {
4
-
return new Date().toISOString();
5
-
}
6
-
7
-
let lastTimestamp = 0;
8
-
9
-
export function generateTid(): string {
10
-
let now = Date.now() * 1000; // microseconds
11
-
if (now <= lastTimestamp) {
12
-
now = lastTimestamp + 1;
13
-
}
14
-
lastTimestamp = now;
15
-
16
-
let tid = "";
17
-
let remaining = now;
18
-
for (let i = 0; i < 13; i++) {
19
-
tid = BASE32_CHARS[remaining & 31] + tid;
20
-
remaining = Math.floor(remaining / 32);
21
-
}
22
-
return tid;
23
-
}
+5
-5
src/server/db/seed.ts
+5
-5
src/server/db/seed.ts
···
1
1
import type { Database } from "bun:sqlite";
2
+
import * as TID from "@atcute/tid";
2
3
import { getDevAccounts } from "../../atproto/env.ts";
3
-
import { generateTid } from "../../lib/tid.ts";
4
4
5
5
export function seedIfEmpty(db: Database): void {
6
6
const count = db.query("SELECT COUNT(*) as n FROM wikis").get() as {
···
10
10
11
11
console.log("Seeding database with sample data...");
12
12
13
-
const homeTid = generateTid();
14
-
const helloTid = generateTid();
15
-
const gettingStartedTid = generateTid();
13
+
const homeTid = TID.now();
14
+
const helloTid = TID.now();
15
+
const gettingStartedTid = TID.now();
16
16
const accounts = getDevAccounts();
17
17
const mockDid = accounts
18
18
? (Object.values(accounts)[0]?.did ?? "did:plc:seed")
···
112
112
113
113
for (let i = 1; i <= 40; i++) {
114
114
const slug = `filler-note-${String(i).padStart(2, "0")}`;
115
-
const tid = generateTid();
115
+
const tid = TID.now();
116
116
const atUri = `at://${mockDid}/wiki.lichen.note/${tid}`;
117
117
db.run(
118
118
"INSERT INTO notes (slug, wiki_slug, title, did, at_uri) VALUES (?, ?, ?, ?, ?)",
+4
-47
src/server/routes/blob.ts
+4
-47
src/server/routes/blob.ts
···
6
6
import { resolveRequestContext } from "../../lib/access.ts";
7
7
import { MIME_TO_EXT } from "../../lib/constants.ts";
8
8
import { formatError } from "../../lib/errors.ts";
9
-
import { resolvePdsEndpoint } from "../../lib/identity.ts";
10
9
import { processImage } from "../../lib/image.ts";
11
10
import { LIMITS } from "../../lib/limits.ts";
11
+
import { fetchVerifiedBlob } from "../../lib/pds-fetch.ts";
12
12
import { getClientIp, rateLimit } from "../../lib/rate-limit.ts";
13
13
14
14
const LOCAL_BLOB_DIR = "data/blobs";
···
129
129
if (limited) return limited;
130
130
131
131
const { did, cid } = params;
132
+
const { data, mimeType } = await fetchVerifiedBlob(did, cid);
132
133
133
-
let pdsEndpoint: string;
134
-
try {
135
-
const resolved = await resolvePdsEndpoint(did);
136
-
if (!resolved) {
137
-
return new Response("PDS not found for DID", { status: 404 });
138
-
}
139
-
pdsEndpoint = resolved;
140
-
} catch {
141
-
return new Response("Failed to resolve DID", { status: 502 });
142
-
}
143
-
144
-
// SSRF guard: only allow HTTPS PDS endpoints in production
145
-
const pdsUrl = new URL(pdsEndpoint);
146
-
const isLocal =
147
-
pdsUrl.hostname === "localhost" || pdsUrl.hostname === "127.0.0.1";
148
-
if (pdsUrl.protocol !== "https:" && !isLocal) {
149
-
return new Response("PDS endpoint must be HTTPS", { status: 502 });
150
-
}
151
-
152
-
const blobUrl = `${pdsEndpoint}/xrpc/com.atproto.sync.getBlob?did=${encodeURIComponent(did)}&cid=${encodeURIComponent(cid)}`;
153
-
154
-
let upstream: Response;
155
-
try {
156
-
upstream = await fetch(blobUrl, {
157
-
signal: AbortSignal.timeout(LIMITS.blobProxy.timeoutMs),
158
-
});
159
-
} catch {
160
-
return new Response("Failed to fetch blob from PDS", {
161
-
status: 502,
162
-
});
163
-
}
164
-
165
-
if (!upstream.ok) {
166
-
return new Response("Blob not found", { status: upstream.status });
167
-
}
168
-
169
-
const contentLength = upstream.headers.get("content-length");
170
-
if (contentLength && Number(contentLength) > LIMITS.blobProxy.maxBytes) {
171
-
return new Response("Blob too large", { status: 413 });
172
-
}
173
-
174
-
const contentType =
175
-
upstream.headers.get("content-type") ?? "application/octet-stream";
176
-
177
-
return new Response(upstream.body, {
134
+
return new Response(data, {
178
135
headers: {
179
-
"Content-Type": contentType,
136
+
"Content-Type": mimeType,
180
137
"Cache-Control": "public, max-age=31536000, immutable",
181
138
"X-Content-Type-Options": "nosniff",
182
139
"Content-Security-Policy": "default-src 'none'; sandbox",
+5
-5
tests/integration/helpers.ts
+5
-5
tests/integration/helpers.ts
···
1
1
import { resolve } from "node:path";
2
+
import * as TID from "@atcute/tid";
2
3
import type { AtpAgent } from "@atproto/api";
3
4
import { IdResolver } from "@atproto/identity";
4
5
import { type Event, Firehose, MemoryRunner } from "@atproto/sync";
···
6
7
import { handleCommitEvent } from "../../src/firehose/handlers.ts";
7
8
import { COLLECTIONS } from "../../src/lib/constants.ts";
8
9
import { createDiff } from "../../src/lib/diff.ts";
9
-
import { generateTid } from "../../src/lib/tid.ts";
10
10
11
11
export interface TestAccount {
12
12
did: string;
···
127
127
visibility = "public",
128
128
): Promise<{ uri: string; cid: string; rkey: string }> {
129
129
const did = agent.session?.did as string;
130
-
const rkey = generateTid();
130
+
const rkey = TID.now();
131
131
const result = await agent.com.atproto.repo.putRecord({
132
132
repo: did,
133
133
collection: COLLECTIONS.wiki,
···
151
151
wikiUri: string,
152
152
): Promise<{ uri: string; cid: string; rkey: string }> {
153
153
const did = agent.session?.did as string;
154
-
const rkey = generateTid();
154
+
const rkey = TID.now();
155
155
const result = await agent.com.atproto.repo.putRecord({
156
156
repo: did,
157
157
collection: COLLECTIONS.note,
···
176
176
parentRevision?: string,
177
177
): Promise<{ uri: string; cid: string }> {
178
178
const did = agent.session?.did as string;
179
-
const rkey = generateTid();
179
+
const rkey = TID.now();
180
180
const record: Record<string, unknown> = {
181
181
$type: COLLECTIONS.noteRevision,
182
182
noteRef: noteUri,
···
197
197
return { uri: result.data.uri, cid: result.data.cid };
198
198
}
199
199
200
-
export { COLLECTIONS, createDiff, generateTid };
200
+
export { COLLECTIONS, createDiff, TID };
+5
-5
tests/integration/roundtrip.test.ts
+5
-5
tests/integration/roundtrip.test.ts
···
15
15
COLLECTIONS,
16
16
createDiff,
17
17
createTestFirehose,
18
-
generateTid,
19
18
putNoteRecord,
20
19
putRevisionRecord,
21
20
putWikiRecord,
22
21
startNetwork,
23
22
type TestNetwork,
23
+
TID,
24
24
waitFor,
25
25
} from "./helpers.ts";
26
26
···
134
134
135
135
await waitFor(() => getWikiByAtUri(wikiUri));
136
136
137
-
const memberRkey = generateTid();
137
+
const memberRkey = TID.now();
138
138
await aliceAgent.com.atproto.repo.putRecord({
139
139
repo: aliceDid,
140
140
collection: COLLECTIONS.membership,
···
274
274
await waitFor(() => getCurrentNote(wikiRkey, "collab-note"), 10000);
275
275
276
276
// Alice adds Bob as admin
277
-
const memberRkey = generateTid();
277
+
const memberRkey = TID.now();
278
278
await aliceAgent.com.atproto.repo.putRecord({
279
279
repo: aliceDid,
280
280
collection: COLLECTIONS.membership,
···
400
400
await waitFor(() => getWikiByAtUri(wikiUri));
401
401
402
402
// Bob creates a member request
403
-
const requestRkey = generateTid();
403
+
const requestRkey = TID.now();
404
404
await bobAgent.com.atproto.repo.putRecord({
405
405
repo: bobDid,
406
406
collection: COLLECTIONS.memberRequest,
···
417
417
expect(req).not.toBeNull();
418
418
419
419
// Alice creates membership for Bob
420
-
const memberRkey = generateTid();
420
+
const memberRkey = TID.now();
421
421
await aliceAgent.com.atproto.repo.putRecord({
422
422
repo: aliceDid,
423
423
collection: COLLECTIONS.membership,
+2
-2
tests/integration/security.test.ts
+2
-2
tests/integration/security.test.ts
···
11
11
import {
12
12
COLLECTIONS,
13
13
createTestFirehose,
14
-
generateTid,
15
14
putWikiRecord,
16
15
startNetwork,
17
16
type TestNetwork,
17
+
TID,
18
18
waitFor,
19
19
} from "./helpers.ts";
20
20
···
70
70
await waitFor(() => getWikiByAtUri(wikiUri));
71
71
72
72
// Bob tries to create a membership on Alice's wiki — should be rejected
73
-
const memberRkey = generateTid();
73
+
const memberRkey = TID.now();
74
74
await bobAgent.com.atproto.repo.putRecord({
75
75
repo: bobDid,
76
76
collection: COLLECTIONS.membership,
+1
-1
tests/lib/blob.test.ts
tests/lib/attachments.test.ts
+1
-1
tests/lib/blob.test.ts
tests/lib/attachments.test.ts
+372
tests/lib/pds-fetch.test.ts
+372
tests/lib/pds-fetch.test.ts
···
1
+
import { beforeEach, describe, expect, mock, test } from "bun:test";
2
+
import * as CID from "@atcute/cid";
3
+
import { LIMITS } from "../../src/lib/limits.ts";
4
+
import { BlobFetchError, fetchVerifiedBlob } from "../../src/lib/pds-fetch.ts";
5
+
6
+
// resolveEndpoint and fetchFn are passed as DI, not module-mocked, to avoid
7
+
// conflicting with other test files that mock @atproto/identity.
8
+
const mockResolvePdsEndpoint = mock(
9
+
async (_did: string): Promise<string | null> => null,
10
+
);
11
+
12
+
const TEST_DID = "did:plc:abc";
13
+
const PDS = "https://pds.example.com";
14
+
15
+
/** Allocate via ArrayBuffer so the type is Uint8Array<ArrayBuffer>, which atcute APIs require. */
16
+
function bytes(input: number | ArrayLike<number>): Uint8Array<ArrayBuffer> {
17
+
if (typeof input === "number") {
18
+
return new Uint8Array(new ArrayBuffer(input));
19
+
}
20
+
const u = new Uint8Array(new ArrayBuffer(input.length));
21
+
u.set(input);
22
+
return u;
23
+
}
24
+
25
+
async function cidFor(data: Uint8Array<ArrayBuffer>): Promise<string> {
26
+
const cid = await CID.create(0x55, data);
27
+
return CID.toString(cid);
28
+
}
29
+
30
+
interface ResponseOpts {
31
+
status?: number;
32
+
contentLength?: string | null;
33
+
chunkSize?: number;
34
+
mimeType?: string;
35
+
}
36
+
37
+
function makeStreamingResponse(
38
+
body: Uint8Array,
39
+
opts: ResponseOpts = {},
40
+
): Response {
41
+
const status = opts.status ?? 200;
42
+
const chunkSize = opts.chunkSize ?? body.byteLength;
43
+
const headers: Record<string, string> = {
44
+
"Content-Type": opts.mimeType ?? "image/png",
45
+
};
46
+
if (opts.contentLength === undefined) {
47
+
headers["Content-Length"] = String(body.byteLength);
48
+
} else if (opts.contentLength !== null) {
49
+
headers["Content-Length"] = opts.contentLength;
50
+
}
51
+
52
+
const stream = new ReadableStream<Uint8Array>({
53
+
start(controller) {
54
+
for (let i = 0; i < body.byteLength; i += chunkSize) {
55
+
controller.enqueue(
56
+
body.subarray(i, Math.min(i + chunkSize, body.byteLength)),
57
+
);
58
+
}
59
+
controller.close();
60
+
},
61
+
});
62
+
63
+
return new Response(stream, { status, headers });
64
+
}
65
+
66
+
function asFetch(fn: () => Promise<Response>): typeof fetch {
67
+
return mock(fn) as unknown as typeof fetch;
68
+
}
69
+
70
+
beforeEach(() => {
71
+
mockResolvePdsEndpoint.mockReset();
72
+
mockResolvePdsEndpoint.mockImplementation(async () => PDS);
73
+
});
74
+
75
+
describe("fetchVerifiedBlob — happy path", () => {
76
+
test("returns data and mime type when CID matches", async () => {
77
+
const data = bytes([1, 2, 3, 4, 5]);
78
+
const cid = await cidFor(data);
79
+
const fetchFn = asFetch(async () =>
80
+
makeStreamingResponse(data, { mimeType: "image/png" }),
81
+
);
82
+
83
+
const result = await fetchVerifiedBlob(
84
+
TEST_DID,
85
+
cid,
86
+
fetchFn,
87
+
mockResolvePdsEndpoint,
88
+
);
89
+
90
+
expect(result.data).toEqual(data);
91
+
expect(result.mimeType).toBe("image/png");
92
+
});
93
+
94
+
test("works across multiple chunks", async () => {
95
+
const data = bytes(1024);
96
+
for (let i = 0; i < data.length; i++) data[i] = i % 256;
97
+
const cid = await cidFor(data);
98
+
const fetchFn = asFetch(async () =>
99
+
makeStreamingResponse(data, { chunkSize: 64 }),
100
+
);
101
+
102
+
const result = await fetchVerifiedBlob(
103
+
TEST_DID,
104
+
cid,
105
+
fetchFn,
106
+
mockResolvePdsEndpoint,
107
+
);
108
+
expect(result.data).toEqual(data);
109
+
});
110
+
});
111
+
112
+
describe("fetchVerifiedBlob — CID verification", () => {
113
+
test("rejects when bytes do not hash to requested CID", async () => {
114
+
// CID is computed for one set of bytes, but the PDS returns different bytes.
115
+
// This is the malicious-PDS-substitutes-content scenario.
116
+
const real = bytes([1, 2, 3]);
117
+
const cid = await cidFor(real);
118
+
const tampered = bytes([9, 9, 9]);
119
+
const fetchFn = asFetch(async () => makeStreamingResponse(tampered));
120
+
121
+
const promise = fetchVerifiedBlob(
122
+
TEST_DID,
123
+
cid,
124
+
fetchFn,
125
+
mockResolvePdsEndpoint,
126
+
);
127
+
expect(promise).rejects.toBeInstanceOf(BlobFetchError);
128
+
expect(promise).rejects.toMatchObject({
129
+
reason: "cid-mismatch",
130
+
statusCode: 502,
131
+
});
132
+
});
133
+
134
+
test("rejects malformed CID input before touching the network", async () => {
135
+
const fetchFn = asFetch(async () => new Response("ok"));
136
+
137
+
const promise = fetchVerifiedBlob(
138
+
TEST_DID,
139
+
"not-a-cid",
140
+
fetchFn,
141
+
mockResolvePdsEndpoint,
142
+
);
143
+
expect(promise).rejects.toMatchObject({
144
+
reason: "invalid-cid",
145
+
statusCode: 400,
146
+
});
147
+
await promise.catch(() => {});
148
+
expect(fetchFn).not.toHaveBeenCalled();
149
+
});
150
+
});
151
+
152
+
describe("fetchVerifiedBlob — size enforcement", () => {
153
+
test("rejects when Content-Length exceeds maxBytes (fast fail)", async () => {
154
+
const data = bytes([1, 2, 3]);
155
+
const cid = await cidFor(data);
156
+
const huge = String(LIMITS.blobProxy.maxBytes + 1);
157
+
const fetchFn = asFetch(async () =>
158
+
makeStreamingResponse(data, { contentLength: huge }),
159
+
);
160
+
161
+
const promise = fetchVerifiedBlob(
162
+
TEST_DID,
163
+
cid,
164
+
fetchFn,
165
+
mockResolvePdsEndpoint,
166
+
);
167
+
expect(promise).rejects.toMatchObject({
168
+
reason: "too-large",
169
+
statusCode: 413,
170
+
});
171
+
});
172
+
173
+
test("rejects when streamed bytes exceed maxBytes despite truthful-looking Content-Length", async () => {
174
+
// Malicious PDS lies: claims small payload, then sends a larger one.
175
+
// This is the security boundary — Content-Length cannot be trusted.
176
+
const oversized = bytes(LIMITS.blobProxy.maxBytes + 1024 * 1024);
177
+
const cid = await cidFor(oversized);
178
+
const fetchFn = asFetch(async () =>
179
+
makeStreamingResponse(oversized, {
180
+
contentLength: "100", // lies
181
+
chunkSize: 256 * 1024,
182
+
}),
183
+
);
184
+
185
+
const promise = fetchVerifiedBlob(
186
+
TEST_DID,
187
+
cid,
188
+
fetchFn,
189
+
mockResolvePdsEndpoint,
190
+
);
191
+
expect(promise).rejects.toMatchObject({
192
+
reason: "too-large",
193
+
statusCode: 413,
194
+
});
195
+
});
196
+
197
+
test("accepts when no Content-Length is sent and body is within cap", async () => {
198
+
const data = bytes([7, 8, 9]);
199
+
const cid = await cidFor(data);
200
+
const fetchFn = asFetch(async () =>
201
+
makeStreamingResponse(data, { contentLength: null }),
202
+
);
203
+
204
+
const result = await fetchVerifiedBlob(
205
+
TEST_DID,
206
+
cid,
207
+
fetchFn,
208
+
mockResolvePdsEndpoint,
209
+
);
210
+
expect(result.data).toEqual(data);
211
+
});
212
+
});
213
+
214
+
describe("fetchVerifiedBlob — SSRF guard", () => {
215
+
test("rejects http (non-localhost) PDS endpoints", async () => {
216
+
mockResolvePdsEndpoint.mockImplementationOnce(
217
+
async () => "http://attacker.example.com",
218
+
);
219
+
const fetchFn = asFetch(async () => new Response("ok"));
220
+
221
+
const data = bytes([1]);
222
+
const cid = await cidFor(data);
223
+
const promise = fetchVerifiedBlob(
224
+
TEST_DID,
225
+
cid,
226
+
fetchFn,
227
+
mockResolvePdsEndpoint,
228
+
);
229
+
230
+
expect(promise).rejects.toMatchObject({
231
+
reason: "non-https-pds",
232
+
statusCode: 502,
233
+
});
234
+
await promise.catch(() => {});
235
+
expect(fetchFn).not.toHaveBeenCalled();
236
+
});
237
+
238
+
test("allows http://localhost (dev)", async () => {
239
+
mockResolvePdsEndpoint.mockImplementationOnce(
240
+
async () => "http://localhost:2583",
241
+
);
242
+
const data = bytes([1, 2]);
243
+
const cid = await cidFor(data);
244
+
const fetchFn = asFetch(async () => makeStreamingResponse(data));
245
+
246
+
const result = await fetchVerifiedBlob(
247
+
TEST_DID,
248
+
cid,
249
+
fetchFn,
250
+
mockResolvePdsEndpoint,
251
+
);
252
+
expect(result.data).toEqual(data);
253
+
});
254
+
255
+
test("allows http://127.0.0.1 (dev)", async () => {
256
+
mockResolvePdsEndpoint.mockImplementationOnce(
257
+
async () => "http://127.0.0.1:2583",
258
+
);
259
+
const data = bytes([3, 4]);
260
+
const cid = await cidFor(data);
261
+
const fetchFn = asFetch(async () => makeStreamingResponse(data));
262
+
263
+
const result = await fetchVerifiedBlob(
264
+
TEST_DID,
265
+
cid,
266
+
fetchFn,
267
+
mockResolvePdsEndpoint,
268
+
);
269
+
expect(result.data).toEqual(data);
270
+
});
271
+
});
272
+
273
+
describe("fetchVerifiedBlob — DID resolution", () => {
274
+
test("rejects when PDS endpoint cannot be resolved", async () => {
275
+
mockResolvePdsEndpoint.mockImplementationOnce(async () => null);
276
+
const fetchFn = asFetch(async () => new Response("ok"));
277
+
const data = bytes([1]);
278
+
const cid = await cidFor(data);
279
+
280
+
const promise = fetchVerifiedBlob(
281
+
TEST_DID,
282
+
cid,
283
+
fetchFn,
284
+
mockResolvePdsEndpoint,
285
+
);
286
+
expect(promise).rejects.toMatchObject({
287
+
reason: "pds-not-found",
288
+
statusCode: 404,
289
+
});
290
+
});
291
+
292
+
test("rejects when DID resolution throws", async () => {
293
+
mockResolvePdsEndpoint.mockImplementationOnce(async () => {
294
+
throw new Error("PLC unreachable");
295
+
});
296
+
const fetchFn = asFetch(async () => new Response("ok"));
297
+
const data = bytes([1]);
298
+
const cid = await cidFor(data);
299
+
300
+
const promise = fetchVerifiedBlob(
301
+
TEST_DID,
302
+
cid,
303
+
fetchFn,
304
+
mockResolvePdsEndpoint,
305
+
);
306
+
expect(promise).rejects.toMatchObject({
307
+
reason: "pds-resolve-failed",
308
+
statusCode: 502,
309
+
});
310
+
});
311
+
});
312
+
313
+
describe("fetchVerifiedBlob — upstream errors", () => {
314
+
test("propagates upstream non-2xx status", async () => {
315
+
const data = bytes([1]);
316
+
const cid = await cidFor(data);
317
+
const fetchFn = asFetch(
318
+
async () => new Response("not found", { status: 404 }),
319
+
);
320
+
321
+
const promise = fetchVerifiedBlob(
322
+
TEST_DID,
323
+
cid,
324
+
fetchFn,
325
+
mockResolvePdsEndpoint,
326
+
);
327
+
expect(promise).rejects.toMatchObject({
328
+
reason: "upstream-error",
329
+
statusCode: 404,
330
+
});
331
+
});
332
+
333
+
test("wraps fetch network failures", async () => {
334
+
const data = bytes([1]);
335
+
const cid = await cidFor(data);
336
+
const fetchFn = asFetch(async () => {
337
+
throw new Error("ECONNREFUSED");
338
+
});
339
+
340
+
const promise = fetchVerifiedBlob(
341
+
TEST_DID,
342
+
cid,
343
+
fetchFn,
344
+
mockResolvePdsEndpoint,
345
+
);
346
+
expect(promise).rejects.toMatchObject({
347
+
reason: "fetch-failed",
348
+
statusCode: 502,
349
+
});
350
+
});
351
+
352
+
test("translates AbortSignal.timeout to a timeout error", async () => {
353
+
const data = bytes([1]);
354
+
const cid = await cidFor(data);
355
+
const fetchFn = asFetch(async () => {
356
+
const err = new Error("aborted");
357
+
err.name = "TimeoutError";
358
+
throw err;
359
+
});
360
+
361
+
const promise = fetchVerifiedBlob(
362
+
TEST_DID,
363
+
cid,
364
+
fetchFn,
365
+
mockResolvePdsEndpoint,
366
+
);
367
+
expect(promise).rejects.toMatchObject({
368
+
reason: "timeout",
369
+
statusCode: 504,
370
+
});
371
+
});
372
+
});
-23
tests/lib/tid.test.ts
-23
tests/lib/tid.test.ts
···
1
-
import { describe, expect, test } from "bun:test";
2
-
import { generateTid } from "../../src/lib/tid.ts";
3
-
4
-
describe("generateTid", () => {
5
-
test("returns 13-character base32 string", () => {
6
-
const tid = generateTid();
7
-
expect(tid).toMatch(/^[2-7a-z]{13}$/);
8
-
});
9
-
10
-
test("successive calls produce unique values", () => {
11
-
const tids = new Set(Array.from({ length: 100 }, () => generateTid()));
12
-
expect(tids.size).toBe(100);
13
-
});
14
-
15
-
test("successive calls are monotonically increasing", () => {
16
-
const a = generateTid();
17
-
const b = generateTid();
18
-
const c = generateTid();
19
-
// Lexicographic ordering matches temporal ordering for base32 TIDs
20
-
expect(a < b).toBe(true);
21
-
expect(b < c).toBe(true);
22
-
});
23
-
});
+4
-5
tests/server/db/queries/note.test.ts
+4
-5
tests/server/db/queries/note.test.ts
···
1
1
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2
+
import * as TID from "@atcute/tid";
2
3
import { applyDiff } from "../../../../src/lib/diff.ts";
3
-
import { generateTid } from "../../../../src/lib/tid.ts";
4
4
import { getDb } from "../../../../src/server/db/index.ts";
5
5
import {
6
6
createNote,
···
18
18
const db = getDb();
19
19
const TEST_DID = "did:plc:mock123";
20
20
21
-
const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${generateTid()}`;
22
-
const revUri = () =>
23
-
`at://${TEST_DID}/wiki.lichen.noteRevision/${generateTid()}`;
21
+
const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${TID.now()}`;
22
+
const revUri = () => `at://${TEST_DID}/wiki.lichen.noteRevision/${TID.now()}`;
24
23
25
24
beforeAll(() => {
26
25
ensureTestWiki();
···
85
84
});
86
85
87
86
test("content is null for notes without current_note", () => {
88
-
const noteAtUri = `at://${TEST_DID}/wiki.lichen.note/${generateTid()}`;
87
+
const noteAtUri = `at://${TEST_DID}/wiki.lichen.note/${TID.now()}`;
89
88
upsertNote(
90
89
"test",
91
90
"read-test-no-content",
+3
-4
tests/server/db/queries/revision.test.ts
+3
-4
tests/server/db/queries/revision.test.ts
···
1
1
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
2
-
import { generateTid } from "../../../../src/lib/tid.ts";
2
+
import * as TID from "@atcute/tid";
3
3
import { getDb } from "../../../../src/server/db/index.ts";
4
4
import { getBlobsByCids } from "../../../../src/server/db/queries/blob.ts";
5
5
import {
···
14
14
const db = getDb();
15
15
const TEST_DID = "did:plc:mock123";
16
16
17
-
const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${generateTid()}`;
18
-
const revUri = () =>
19
-
`at://${TEST_DID}/wiki.lichen.noteRevision/${generateTid()}`;
17
+
const noteUri = () => `at://${TEST_DID}/wiki.lichen.note/${TID.now()}`;
18
+
const revUri = () => `at://${TEST_DID}/wiki.lichen.noteRevision/${TID.now()}`;
20
19
21
20
beforeAll(() => {
22
21
ensureTestWiki();