juprodh.me / lichen.wiki
🌿 Collaborative wiki on ATProto lichen.wiki
atproto
14
fork

Configure Feed

Select the types of activity you want to include in your feed.

Move blob management and export

juprodh c5b1aa05 1c7e6bb8

+192 -140
-10
src/lib/blob.ts
··· 1 1 import type { RevisionBlob } from "../atproto/pds.ts"; 2 - import { insertBlob } from "../server/db/queries/index.ts"; 3 2 4 3 const BLOB_URL_RE = /!\[[^\]]*\]\(\/blob\/(did:[^/]+)\/([^)\s]+)\)/g; 5 4 ··· 64 63 } 65 64 return blobs; 66 65 } 67 - 68 - export function persistBlobs( 69 - blobs: RevisionBlob[], 70 - revisionAtUri: string, 71 - ): void { 72 - for (const blob of blobs) { 73 - insertBlob(blob.ref.$link, revisionAtUri, blob.mimeType, blob.ref.$link); 74 - } 75 - }
+9 -7
src/lib/import-export/export.ts
··· 1 1 import { zipSync } from "fflate"; 2 2 import { getBlobsByCids } from "../../server/db/queries/blob.ts"; 3 - import { getCurrentNote, listNotes } from "../../server/db/queries/index.ts"; 3 + import { 4 + getCurrentNote, 5 + getSidebarNotes, 6 + listNotesWithContent, 7 + } from "../../server/db/queries/index.ts"; 4 8 import { MIME_TO_EXT } from "../constants.ts"; 5 9 import { resolvePdsEndpoint } from "../identity.ts"; 6 10 import { rewriteForExport } from "./markdown-transform.ts"; ··· 13 17 * Blob images are fetched and saved as attachments/{cid}.{ext}. 14 18 */ 15 19 export async function exportWikiZip(wikiSlug: string): Promise<Uint8Array> { 16 - const notes = listNotes(wikiSlug); 20 + const notes = listNotesWithContent(wikiSlug); 17 21 const slugToTitle = new Map<string, string>(); 18 22 const noteContents = new Map<string, string>(); 19 23 20 24 for (const note of notes) { 21 25 slugToTitle.set(note.slug, note.title); 22 - const current = getCurrentNote(wikiSlug, note.slug); 23 - if (current) { 24 - noteContents.set(note.slug, current.content); 26 + if (note.content !== null) { 27 + noteContents.set(note.slug, note.content); 25 28 } 26 29 } 27 30 ··· 139 142 const current = getCurrentNote(wikiSlug, noteSlug); 140 143 if (!current) return null; 141 144 142 - const notes = listNotes(wikiSlug); 143 145 const slugToTitle = new Map<string, string>(); 144 - for (const note of notes) { 146 + for (const note of getSidebarNotes(wikiSlug)) { 145 147 slugToTitle.set(note.slug, note.title); 146 148 } 147 149
+2 -2
src/lib/import-export/import.ts
··· 2 2 import type { getAgent } from "../../atproto/session.ts"; 3 3 import { createNote } from "../../server/db/queries/index.ts"; 4 4 import type { RequestContext } from "../access.ts"; 5 - import { type BlobMeta, buildBlobsForContent, persistBlobs } from "../blob.ts"; 5 + import { type BlobMeta, buildBlobsForContent } from "../blob.ts"; 6 6 import { createDiff } from "../diff.ts"; 7 7 import type { Messages } from "../i18n/index.ts"; 8 8 import { processImage } from "../image.ts"; ··· 94 94 did, 95 95 content, 96 96 "Imported", 97 + blobs.length > 0 ? blobs : undefined, 97 98 ); 98 - persistBlobs(blobs, revisionAtUri); 99 99 } 100 100 101 101 return { wikiSlug, noteCount: notes.length, warnings };
+2 -3
src/lib/orchestrators/note.ts
··· 17 17 type BlobMeta, 18 18 buildBlobsForContent, 19 19 parseBlobMetadata, 20 - persistBlobs, 21 20 } from "../blob.ts"; 22 21 import { COLLECTIONS } from "../constants.ts"; 23 22 import { createDiff } from "../diff.ts"; ··· 163 162 did, 164 163 fields.content, 165 164 fields.message, 165 + blobs.length > 0 ? blobs : undefined, 166 166 ); 167 - persistBlobs(blobs, revisionAtUri); 168 167 169 168 return { noteSlug }; 170 169 } ··· 248 247 did, 249 248 fields.message, 250 249 newTitle, 250 + blobs.length > 0 ? blobs : undefined, 251 251 ); 252 - persistBlobs(blobs, revisionAtUri); 253 252 } 254 253 255 254 /**
-14
src/server/db/queries/blob.ts
··· 1 1 import { getDb } from "../index.ts"; 2 2 import type { BlobRow } from "../types.ts"; 3 3 4 - export function insertBlob( 5 - cid: string, 6 - revisionAtUri: string, 7 - mimeType: string, 8 - storageKey: string, 9 - ): void { 10 - const db = getDb(); 11 - db.run( 12 - `INSERT OR IGNORE INTO blobs (cid, revision_at_uri, mime_type, storage_key) 13 - VALUES (?, ?, ?, ?)`, 14 - [cid, revisionAtUri, mimeType, storageKey], 15 - ); 16 - } 17 - 18 4 export function getBlobsByCids(cids: string[]): BlobRow[] { 19 5 if (cids.length === 0) return []; 20 6 const db = getDb();
+1 -2
src/server/db/queries/index.ts
··· 1 1 // Barrel re-export — all query functions accessible from "db/queries" 2 2 export type { MembershipRow, RequestRow, WikiRow } from "../types.ts"; 3 - export { insertBlob } from "./blob.ts"; 4 3 export { 5 4 deleteBookmarkByUri, 6 5 deleteBookmarkByWiki, ··· 33 32 getNoteBySlug, 34 33 getNoteWithCurrent, 35 34 getSidebarNotes, 36 - listNotes, 35 + listNotesWithContent, 37 36 saveNoteEdit, 38 37 searchNotes, 39 38 upsertNote,
+23 -4
src/server/db/queries/note.ts
··· 1 + import type { RevisionBlob } from "../../../atproto/pds.ts"; 1 2 import { createDiff } from "../../../lib/diff.ts"; 2 3 import { escapeLikePattern } from "../../../lib/html.ts"; 3 4 import { getDb } from "../index.ts"; ··· 27 28 export function getSidebarNotes( 28 29 wikiSlug: string, 29 30 ): { slug: string; title: string }[] { 30 - return listNotes(wikiSlug).map((n) => ({ slug: n.slug, title: n.title })); 31 + const db = getDb(); 32 + return db 33 + .query( 34 + "SELECT slug, title FROM notes WHERE wiki_slug = ? ORDER BY created_at DESC", 35 + ) 36 + .all(wikiSlug) as { slug: string; title: string }[]; 31 37 } 32 38 33 - export function listNotes(wikiSlug: string): NoteRow[] { 39 + /** Fetch all notes with their current content in a single query. */ 40 + export function listNotesWithContent( 41 + wikiSlug: string, 42 + ): { slug: string; title: string; content: string | null }[] { 34 43 const db = getDb(); 35 44 return db 36 - .query("SELECT * FROM notes WHERE wiki_slug = ? ORDER BY created_at DESC") 37 - .all(wikiSlug) as NoteRow[]; 45 + .query( 46 + `SELECT n.slug, n.title, c.content 47 + FROM notes n 48 + LEFT JOIN current_note c ON c.note_at_uri = n.at_uri 49 + WHERE n.wiki_slug = ? 50 + ORDER BY n.created_at DESC`, 51 + ) 52 + .all(wikiSlug) as { slug: string; title: string; content: string | null }[]; 38 53 } 39 54 40 55 export interface NoteSearchResult extends NoteRow { ··· 160 175 did: string, 161 176 initialContent: string, 162 177 message?: string, 178 + blobs?: RevisionBlob[], 163 179 ): void { 164 180 const db = getDb(); 165 181 const diff = createDiff("", initialContent); ··· 178 194 diff, 179 195 message: message ?? null, 180 196 newContent: initialContent, 197 + ...(blobs ? { blobs } : {}), 181 198 }); 182 199 })(); 183 200 ··· 196 213 did: string, 197 214 message?: string, 198 215 newTitle?: string, 216 + blobs?: RevisionBlob[], 199 217 ): void { 200 218 const db = getDb(); 201 219 ··· 234 252 diff, 235 253 message: message ?? null, 236 254 newContent, 255 + ...(blobs ? { blobs } : {}), 237 256 }); 238 257 })(); 239 258
+12 -77
tests/server/db/queries/blob.test.ts
··· 1 1 import { afterAll, beforeAll, describe, expect, test } from "bun:test"; 2 2 import { getDb } from "../../../../src/server/db/index.ts"; 3 - import { 4 - getBlobsByCids, 5 - insertBlob, 6 - } from "../../../../src/server/db/queries/blob.ts"; 3 + import { getBlobsByCids } from "../../../../src/server/db/queries/blob.ts"; 7 4 8 5 const db = getDb(); 9 6 ··· 11 8 const WIKI_SLUG = "blob-test-wiki"; 12 9 const WIKI_AT_URI = `at://${TEST_DID}/wiki.lichen.wiki/${WIKI_SLUG}`; 13 10 const NOTE_AT_URI = `at://${TEST_DID}/wiki.lichen.note/note1`; 14 - const REV_URI_1 = `at://${TEST_DID}/wiki.lichen.noteRevision/rev1`; 15 - const REV_URI_2 = `at://${TEST_DID}/wiki.lichen.noteRevision/rev2`; 16 - 17 - const TEST_CIDS = [ 18 - "bafytest-blob-1", 19 - "bafytest-blob-2", 20 - "bafytest-blob-3", 21 - "bafytest-blob-dup", 22 - ]; 11 + const REV_URI = `at://${TEST_DID}/wiki.lichen.noteRevision/rev1`; 23 12 24 13 beforeAll(() => { 25 14 db.run( ··· 32 21 ); 33 22 db.run( 34 23 "INSERT INTO revisions (note_at_uri, did, at_uri, diff) VALUES (?, ?, ?, ?)", 35 - [NOTE_AT_URI, TEST_DID, REV_URI_1, ""], 24 + [NOTE_AT_URI, TEST_DID, REV_URI, ""], 36 25 ); 37 26 db.run( 38 - "INSERT INTO revisions (note_at_uri, did, at_uri, diff) VALUES (?, ?, ?, ?)", 39 - [NOTE_AT_URI, TEST_DID, REV_URI_2, ""], 27 + "INSERT OR IGNORE INTO blobs (cid, revision_at_uri, mime_type, storage_key) VALUES (?, ?, ?, ?)", 28 + ["bafytest-blob-1", REV_URI, "image/webp", "bafytest-blob-1"], 29 + ); 30 + db.run( 31 + "INSERT OR IGNORE INTO blobs (cid, revision_at_uri, mime_type, storage_key) VALUES (?, ?, ?, ?)", 32 + ["bafytest-blob-2", REV_URI, "image/gif", "bafytest-blob-2"], 40 33 ); 41 34 }); 42 35 43 36 afterAll(() => { 44 - for (const cid of TEST_CIDS) { 45 - db.run("DELETE FROM blobs WHERE cid = ?", [cid]); 46 - } 47 37 db.run("DELETE FROM wikis WHERE slug = ?", [WIKI_SLUG]); 48 38 }); 49 39 50 - describe("insertBlob", () => { 51 - test("inserts a blob row", () => { 52 - insertBlob( 53 - "bafytest-blob-1", 54 - REV_URI_1, 55 - "image/png", 56 - "data/blobs/test1.png", 57 - ); 58 - 59 - const row = db 60 - .query("SELECT * FROM blobs WHERE cid = ?") 61 - .get("bafytest-blob-1") as Record<string, unknown> | null; 62 - expect(row).not.toBeNull(); 63 - expect(row?.["mime_type"]).toBe("image/png"); 64 - expect(row?.["storage_key"]).toBe("data/blobs/test1.png"); 65 - }); 66 - 67 - test("INSERT OR IGNORE on duplicate CID", () => { 68 - insertBlob( 69 - "bafytest-blob-dup", 70 - REV_URI_1, 71 - "image/jpeg", 72 - "data/blobs/first.jpg", 73 - ); 74 - // Insert again with different values -- should be ignored 75 - insertBlob( 76 - "bafytest-blob-dup", 77 - REV_URI_2, 78 - "image/png", 79 - "data/blobs/second.png", 80 - ); 81 - 82 - const row = db 83 - .query("SELECT * FROM blobs WHERE cid = ?") 84 - .get("bafytest-blob-dup") as Record<string, unknown> | null; 85 - // Original values preserved 86 - expect(row?.["mime_type"]).toBe("image/jpeg"); 87 - expect(row?.["storage_key"]).toBe("data/blobs/first.jpg"); 88 - }); 89 - }); 90 - 91 40 describe("getBlobsByCids", () => { 92 41 test("empty array returns empty array", () => { 93 42 expect(getBlobsByCids([])).toEqual([]); 94 43 }); 95 44 96 45 test("returns matching blobs", () => { 97 - insertBlob( 98 - "bafytest-blob-2", 99 - REV_URI_1, 100 - "image/webp", 101 - "data/blobs/test2.webp", 102 - ); 103 - insertBlob( 104 - "bafytest-blob-3", 105 - REV_URI_1, 106 - "image/gif", 107 - "data/blobs/test3.gif", 108 - ); 109 - 110 - const blobs = getBlobsByCids(["bafytest-blob-2", "bafytest-blob-3"]); 46 + const blobs = getBlobsByCids(["bafytest-blob-1", "bafytest-blob-2"]); 111 47 expect(blobs).toHaveLength(2); 112 48 const cids = blobs.map((b) => b.cid); 49 + expect(cids).toContain("bafytest-blob-1"); 113 50 expect(cids).toContain("bafytest-blob-2"); 114 - expect(cids).toContain("bafytest-blob-3"); 115 51 }); 116 52 117 53 test("nonexistent CIDs return empty array", () => { 118 - const blobs = getBlobsByCids(["bafynonexistent1", "bafynonexistent2"]); 119 - expect(blobs).toEqual([]); 54 + expect(getBlobsByCids(["bafynonexistent1"])).toEqual([]); 120 55 }); 121 56 });
+23 -21
tests/server/db/queries/note.test.ts
··· 8 8 getCurrentNote, 9 9 getNoteByAtUri, 10 10 getNoteBySlug, 11 - listNotes, 11 + listNotesWithContent, 12 12 saveNoteEdit, 13 13 searchNotes, 14 14 upsertNote, ··· 29 29 cleanupNotes("test", "search-test-*"); 30 30 cleanupNotes("test", "upsert-test-*"); 31 31 cleanupNotes("test", "delete-test-*"); 32 - cleanupNotes("test", "list-test-*"); 33 32 cleanupNotes("test", "read-test-*"); 34 33 35 34 // Create notes for read-only tests ··· 68 67 cleanupNotes("test", "search-test-*"); 69 68 cleanupNotes("test", "upsert-test-*"); 70 69 cleanupNotes("test", "delete-test-*"); 71 - cleanupNotes("test", "list-test-*"); 72 70 cleanupNotes("test", "read-test-*"); 73 71 }); 74 72 75 - describe("listNotes", () => { 76 - test("returns notes for a wiki", () => { 77 - const notes = listNotes("test"); 73 + describe("listNotesWithContent", () => { 74 + test("returns notes with content for a wiki", () => { 75 + const notes = listNotesWithContent("test"); 78 76 expect(notes.length).toBeGreaterThanOrEqual(3); 79 - const slugs = notes.map((n) => n.slug); 80 - expect(slugs).toContain("read-test-home"); 81 - expect(slugs).toContain("read-test-hello"); 82 - expect(slugs).toContain("read-test-guide"); 83 - }); 84 - 85 - test("returns notes with titles", () => { 86 - const notes = listNotes("test"); 87 77 const hello = notes.find((n) => n.slug === "read-test-hello"); 78 + expect(hello).toBeDefined(); 88 79 expect(hello?.title).toBe("Hello World"); 80 + expect(hello?.content).toContain("# Hello World"); 89 81 }); 90 82 91 83 test("returns empty array for nonexistent wiki", () => { 92 - const notes = listNotes("nonexistent"); 93 - expect(notes).toEqual([]); 84 + expect(listNotesWithContent("nonexistent")).toEqual([]); 94 85 }); 95 86 96 - test("notes have correct wiki_slug", () => { 97 - const notes = listNotes("test"); 98 - for (const note of notes) { 99 - expect(note.wiki_slug).toBe("test"); 100 - } 87 + test("content is null for notes without current_note", () => { 88 + const noteAtUri = `at://${TEST_DID}/wiki.lichen.note/${generateTid()}`; 89 + upsertNote( 90 + "test", 91 + "read-test-no-content", 92 + "No Content", 93 + TEST_DID, 94 + noteAtUri, 95 + "2026-01-01T00:00:00.000Z", 96 + ); 97 + const notes = listNotesWithContent("test"); 98 + const found = notes.find((n) => n.slug === "read-test-no-content"); 99 + expect(found).toBeDefined(); 100 + expect(found?.content).toBeNull(); 101 + // cleanup 102 + getDb().run("DELETE FROM notes WHERE at_uri = ?", [noteAtUri]); 101 103 }); 102 104 }); 103 105
+120
tests/server/db/queries/revision.test.ts
··· 1 1 import { afterAll, beforeAll, describe, expect, test } from "bun:test"; 2 2 import { generateTid } from "../../../../src/lib/tid.ts"; 3 3 import { getDb } from "../../../../src/server/db/index.ts"; 4 + import { getBlobsByCids } from "../../../../src/server/db/queries/blob.ts"; 4 5 import { 5 6 capSnapshots, 6 7 createNote, ··· 21 22 ensureTestWiki(); 22 23 cleanupNotes("test", "backlink-test-*"); 23 24 cleanupNotes("test", "snapshot-test-*"); 25 + cleanupNotes("test", "blob-test-*"); 24 26 }); 25 27 26 28 afterAll(() => { 27 29 cleanupNotes("test", "backlink-test-*"); 28 30 cleanupNotes("test", "snapshot-test-*"); 31 + cleanupNotes("test", "blob-test-*"); 29 32 }); 30 33 31 34 describe("backlinks", () => { ··· 249 252 expect(getSnapshots("at://fake/uri")).toEqual([]); 250 253 }); 251 254 }); 255 + 256 + describe("blobs in appendRevisionTx", () => { 257 + test("createNote persists blobs", () => { 258 + const nUri = noteUri(); 259 + createNote( 260 + nUri, 261 + revUri(), 262 + "test", 263 + "blob-test-create", 264 + "Blob Create", 265 + TEST_DID, 266 + "content", 267 + undefined, 268 + [ 269 + { 270 + $type: "blob", 271 + ref: { $link: "bafyrev-create-1" }, 272 + mimeType: "image/png", 273 + size: 1000, 274 + }, 275 + { 276 + $type: "blob", 277 + ref: { $link: "bafyrev-create-2" }, 278 + mimeType: "image/webp", 279 + size: 2000, 280 + }, 281 + ], 282 + ); 283 + const blobs = getBlobsByCids(["bafyrev-create-1", "bafyrev-create-2"]); 284 + expect(blobs).toHaveLength(2); 285 + expect(blobs.map((b) => b.mime_type).sort()).toEqual([ 286 + "image/png", 287 + "image/webp", 288 + ]); 289 + }); 290 + 291 + test("saveNoteEdit persists blobs", () => { 292 + createNote( 293 + noteUri(), 294 + revUri(), 295 + "test", 296 + "blob-test-edit", 297 + "Blob Edit", 298 + TEST_DID, 299 + "v1", 300 + ); 301 + saveNoteEdit( 302 + revUri(), 303 + "test", 304 + "blob-test-edit", 305 + "v2", 306 + TEST_DID, 307 + undefined, 308 + undefined, 309 + [ 310 + { 311 + $type: "blob", 312 + ref: { $link: "bafyrev-edit-1" }, 313 + mimeType: "image/gif", 314 + size: 500, 315 + }, 316 + ], 317 + ); 318 + const blobs = getBlobsByCids(["bafyrev-edit-1"]); 319 + expect(blobs).toHaveLength(1); 320 + expect(blobs[0]?.mime_type).toBe("image/gif"); 321 + }); 322 + 323 + test("createNote without blobs inserts no blob rows", () => { 324 + createNote( 325 + noteUri(), 326 + revUri(), 327 + "test", 328 + "blob-test-none", 329 + "No Blobs", 330 + TEST_DID, 331 + "content", 332 + ); 333 + const blobs = getBlobsByCids(["bafyrev-none-should-not-exist"]); 334 + expect(blobs).toEqual([]); 335 + }); 336 + 337 + test("duplicate blob CID is ignored", () => { 338 + const nUri = noteUri(); 339 + const blob = { 340 + $type: "blob" as const, 341 + ref: { $link: "bafyrev-dup-1" }, 342 + mimeType: "image/jpeg", 343 + size: 300, 344 + }; 345 + createNote( 346 + nUri, 347 + revUri(), 348 + "test", 349 + "blob-test-dup", 350 + "Blob Dup", 351 + TEST_DID, 352 + "v1", 353 + undefined, 354 + [blob], 355 + ); 356 + // Same CID on a second revision -- INSERT OR IGNORE should keep the original 357 + saveNoteEdit( 358 + revUri(), 359 + "test", 360 + "blob-test-dup", 361 + "v2", 362 + TEST_DID, 363 + undefined, 364 + undefined, 365 + [{ ...blob, mimeType: "image/png", size: 999 }], 366 + ); 367 + const blobs = getBlobsByCids(["bafyrev-dup-1"]); 368 + expect(blobs).toHaveLength(1); 369 + expect(blobs[0]?.mime_type).toBe("image/jpeg"); 370 + }); 371 + });