-3

deno.jsonc

reviewed

··· 11 11 "@atcute/lexicons": "npm:@atcute/lexicons@^1.2.7", 12 12 "@atcute/oauth-browser-client": "npm:@atcute/oauth-browser-client@^3.0.0", 13 13 "@automerge/automerge": "npm:@automerge/automerge@^3.2.3", 14 14 - "@automerge/automerge-repo": "npm:@automerge/automerge-repo@^2.5.1", 15 15 - "@automerge/automerge-repo-network-broadcastchannel": "npm:@automerge/automerge-repo-network-broadcastchannel@^2.5.1", 16 16 - "@automerge/automerge-repo-storage-indexeddb": "npm:@automerge/automerge-repo-storage-indexeddb@^2.5.1", 17 14 "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.9.5", 18 15 "@char/cbor": "jsr:@char/cbor@^0.1.4", 19 16 "@codemirror/autocomplete": "npm:@codemirror/autocomplete@^6.20.0",

-371

src/components/output/bytes/automerge-repo-server/element.js

reviewed

··· 1 1 - import * as Automerge from "@automerge/automerge"; 2 2 - import { decodeCBOR, encodeCBOR } from "@char/cbor"; 3 3 - import bs58check from "bs58check"; 4 4 - 5 5 - import { DiffuseElement } from "@common/element.js"; 6 6 - import { outputManager } from "../../common.js"; 7 7 - 8 8 - /** 9 9 - * @import { DocumentId, PeerId } from "@automerge/automerge-repo" 10 10 - * @import { OutputElement, OutputManager, OutputManagerProperties } from "../../types.d.ts" 11 11 - */ 12 12 - 13 13 - /** 14 14 - * @typedef {{ collection: Uint8Array }} CollectionDocument 15 15 - */ 16 16 - 17 17 - const DOC_IDS_STORAGE_KEY = "diffuse/output/automerge-repo-server/doc-ids"; 18 18 - const PROTOCOL_VERSION = "1"; 19 19 - 20 20 - const COLLECTIONS = /** @type {const} */ ([ 21 21 - "facets", 22 22 - "playlistItems", 23 23 - "themes", 24 24 - "tracks", 25 25 - ]); 26 26 - 27 27 - //////////////////////////////////////////// 28 28 - // ELEMENT 29 29 - //////////////////////////////////////////// 30 30 - 31 31 - /** 32 32 - * Syncs raw Automerge bytes with an automerge-repo sync server 33 33 - * over WebSocket, without depending on automerge-repo as a client. 34 34 - * 35 35 - * Uses the automerge-repo wire protocol (CBOR-framed messages with 36 36 - * join handshake) and the standard Automerge sync algorithm. 37 37 - * 38 38 - * @implements {OutputElement<Uint8Array>} 39 39 - */ 40 40 - class AutomergeRepoServerOutput extends DiffuseElement { 41 41 - static NAME = "diffuse/output/bytes/automerge-repo-server"; 42 42 - 43 43 - /** @type {WebSocket | undefined} */ 44 44 - #ws; 45 45 - 46 46 - /** @type {PeerId} */ 47 47 - #peerId = /** @type {PeerId} */ (`diffuse-${crypto.randomUUID()}`); 48 48 - 49 49 - /** @type {PeerId | undefined} */ 50 50 - #serverPeerId; 51 51 - 52 52 - /** @type {Record<string, Automerge.Doc<CollectionDocument>>} */ 53 53 - #docs = {}; 54 54 - 55 55 - /** @type {Record<string, Automerge.SyncState>} */ 56 56 - #syncStates = {}; 57 57 - 58 58 - /** @type {Record<string, DocumentId>} */ 59 59 - #docIds = {}; 60 60 - 61 61 - #manager; 62 62 - 63 63 - constructor() { 64 64 - super(); 65 65 - 66 66 - this.#ensureDocs(); 67 67 - 68 68 - /** @type {OutputManagerProperties<Uint8Array>} */ 69 69 - const properties = { 70 70 - facets: { 71 71 - empty: () => this.#getBytes("facets"), 72 72 - get: async () => this.#getBytes("facets"), 73 73 - put: async (data) => this.#putBytes("facets", data), 74 74 - }, 75 75 - init: () => this.whenConnected(), 76 76 - playlistItems: { 77 77 - empty: () => this.#getBytes("playlistItems"), 78 78 - get: async () => this.#getBytes("playlistItems"), 79 79 - put: async (data) => this.#putBytes("playlistItems", data), 80 80 - }, 81 81 - themes: { 82 82 - empty: () => this.#getBytes("themes"), 83 83 - get: async () => this.#getBytes("themes"), 84 84 - put: async (data) => this.#putBytes("themes", data), 85 85 - }, 86 86 - tracks: { 87 87 - empty: () => this.#getBytes("tracks"), 88 88 - get: async () => this.#getBytes("tracks"), 89 89 - put: async (data) => this.#putBytes("tracks", data), 90 90 - }, 91 91 - }; 92 92 - 93 93 - this.#manager = outputManager(properties); 94 94 - 95 95 - this.facets = this.#manager.facets; 96 96 - this.playlistItems = this.#manager.playlistItems; 97 97 - this.themes = this.#manager.themes; 98 98 - this.tracks = this.#manager.tracks; 99 99 - this.ready = () => true; 100 100 - } 101 101 - 102 102 - // LIFECYCLE 103 103 - 104 104 - /** 105 105 - * @override 106 106 - */ 107 107 - connectedCallback() { 108 108 - super.connectedCallback(); 109 109 - this.#loadDocIds(); 110 110 - this.#connect(); 111 111 - } 112 112 - 113 113 - /** 114 114 - * @override 115 115 - */ 116 116 - disconnectedCallback() { 117 117 - super.disconnectedCallback(); 118 118 - this.#ws?.close(); 119 119 - this.#ws = undefined; 120 120 - } 121 121 - 122 122 - // DOCUMENT MANAGEMENT 123 123 - 124 124 - #loadDocIds() { 125 125 - const namespace = this.getAttribute("namespace") ?? "automerge-repo-server"; 126 126 - const storageKey = `${DOC_IDS_STORAGE_KEY}/${namespace}`; 127 127 - const stored = localStorage.getItem(storageKey); 128 128 - 129 129 - if (stored) { 130 130 - this.#docIds = JSON.parse(stored); 131 131 - } 132 132 - 133 133 - // Ensure every collection has a document ID 134 134 - for (const name of COLLECTIONS) { 135 135 - if (!this.#docIds[name]) { 136 136 - const bytes = crypto.getRandomValues(new Uint8Array(16)); 137 137 - 138 138 - // Set UUID v4 version and variant bits 139 139 - bytes[6] = (bytes[6] & 0x0f) | 0x40; 140 140 - bytes[8] = (bytes[8] & 0x3f) | 0x80; 141 141 - 142 142 - const docId = /** @type {DocumentId} */ (bs58check.encode(bytes)); 143 143 - const url = `automerge:${docId}`; 144 144 - 145 145 - this.#docIds[name] = docId; 146 146 - } 147 147 - } 148 148 - 149 149 - localStorage.setItem(storageKey, JSON.stringify(this.#docIds)); 150 150 - } 151 151 - 152 152 - #ensureDocs() { 153 153 - for (const name of COLLECTIONS) { 154 154 - if (!this.#docs[name]) { 155 155 - this.#docs[name] = Automerge.init(); 156 156 - } 157 157 - if (!this.#syncStates[name]) { 158 158 - this.#syncStates[name] = Automerge.initSyncState(); 159 159 - } 160 160 - } 161 161 - } 162 162 - 163 163 - /** 164 164 - * @param {string} name 165 165 - * @returns {Uint8Array} 166 166 - */ 167 167 - #getBytes(name) { 168 168 - const doc = this.#docs[name]; 169 169 - if (doc) return Automerge.save(doc); 170 170 - return new Uint8Array(); 171 171 - } 172 172 - 173 173 - /** 174 174 - * @param {string} name 175 175 - * @param {Uint8Array} data 176 176 - */ 177 177 - #putBytes(name, data) { 178 178 - if (data.byteLength > 0) { 179 179 - this.#docs[name] = Automerge.load(data); 180 180 - } else { 181 181 - this.#docs[name] = Automerge.init(); 182 182 - } 183 183 - 184 184 - this.#syncDoc(name); 185 185 - } 186 186 - 187 187 - // WEBSOCKET CONNECTION 188 188 - 189 189 - #connect() { 190 190 - const url = this.getAttribute("url"); 191 191 - if (!url) return; 192 192 - 193 193 - const ws = new WebSocket(url); 194 194 - ws.binaryType = "arraybuffer"; 195 195 - this.#ws = ws; 196 196 - 197 197 - ws.addEventListener("open", () => { 198 198 - this.#sendJoin(); 199 199 - }); 200 200 - 201 201 - ws.addEventListener("message", (event) => { 202 202 - const msg = this.#cborDecode(new Uint8Array(event.data)); 203 203 - this.#handleMessage(msg); 204 204 - }); 205 205 - 206 206 - ws.addEventListener("close", () => { 207 207 - this.#serverPeerId = undefined; 208 208 - this.#scheduleReconnect(); 209 209 - }); 210 210 - 211 211 - ws.addEventListener("error", () => { 212 212 - ws.close(); 213 213 - }); 214 214 - } 215 215 - 216 216 - #scheduleReconnect() { 217 217 - if (!this.isConnected) return; 218 218 - setTimeout(() => { 219 219 - if (this.isConnected) this.#connect(); 220 220 - }, 5000); 221 221 - } 222 222 - 223 223 - // PROTOCOL 224 224 - 225 225 - #sendJoin() { 226 226 - this.#send({ 227 227 - type: "join", 228 228 - senderId: this.#peerId, 229 229 - peerMetadata: { storageId: undefined, isEphemeral: true }, 230 230 - supportedProtocolVersions: [PROTOCOL_VERSION], 231 231 - }); 232 232 - } 233 233 - 234 234 - /** 235 235 - * @param {any} msg 236 236 - */ 237 237 - #handleMessage(msg) { 238 238 - switch (msg.type) { 239 239 - case "peer": 240 240 - this.#serverPeerId = msg.senderId; 241 241 - this.#syncAllDocs(); 242 242 - break; 243 243 - 244 244 - case "sync": 245 245 - case "request": 246 246 - this.#handleSyncMessage(msg); 247 247 - break; 248 248 - 249 249 - case "doc-unavailable": 250 250 - // Server doesn't have this doc; that's fine, we'll push ours 251 251 - break; 252 252 - 253 253 - case "error": 254 254 - console.error("[automerge-repo-server]", msg.message); 255 255 - break; 256 256 - } 257 257 - } 258 258 - 259 259 - /** 260 260 - * @param {{ documentId: DocumentId, data: Uint8Array }} msg 261 261 - */ 262 262 - #handleSyncMessage(msg) { 263 263 - const name = this.#nameForDocId(msg.documentId); 264 264 - if (!name) return; 265 265 - 266 266 - const doc = this.#docs[name] ?? Automerge.init(); 267 267 - const syncState = this.#syncStates[name] ?? Automerge.initSyncState(); 268 268 - 269 269 - const [newDoc, newSyncState] = Automerge.receiveSyncMessage( 270 270 - doc, 271 271 - syncState, 272 272 - msg.data, 273 273 - ); 274 274 - 275 275 - this.#docs[name] = newDoc; 276 276 - this.#syncStates[name] = newSyncState; 277 277 - 278 278 - // Update the output manager signal with fresh bytes 279 279 - // @ts-ignore Not sure what type to use here 280 280 - this.#manager.signals[name].value = Automerge.save(newDoc); 281 281 - 282 282 - // Continue the sync round-trip 283 283 - this.#syncDoc(name); 284 284 - } 285 285 - 286 286 - // SYNC 287 287 - 288 288 - #syncAllDocs() { 289 289 - for (const name of COLLECTIONS) { 290 290 - this.#syncDoc(name); 291 291 - } 292 292 - } 293 293 - 294 294 - /** 295 295 - * @param {string} name 296 296 - */ 297 297 - #syncDoc(name) { 298 298 - if (!this.#serverPeerId) return; 299 299 - 300 300 - const doc = this.#docs[name]; 301 301 - const syncState = this.#syncStates[name] ?? Automerge.initSyncState(); 302 302 - 303 303 - if (!doc) return; 304 304 - 305 305 - const [newSyncState, syncMessage] = Automerge.generateSyncMessage( 306 306 - doc, 307 307 - syncState, 308 308 - ); 309 309 - 310 310 - this.#syncStates[name] = newSyncState; 311 311 - 312 312 - if (syncMessage) { 313 313 - this.#send({ 314 314 - type: "sync", 315 315 - senderId: this.#peerId, 316 316 - targetId: this.#serverPeerId, 317 317 - documentId: this.#docIds[name], 318 318 - data: syncMessage, 319 319 - }); 320 320 - } 321 321 - } 322 322 - 323 323 - // HELPERS 324 324 - 325 325 - /** 326 326 - * @template [T=unknown] 327 327 - * @param {Uint8Array} data 328 328 - * @returns {T} 329 329 - */ 330 330 - #cborDecode(data) { 331 331 - return /** @type {T} */ (decodeCBOR(data)); 332 332 - } 333 333 - 334 334 - /** 335 335 - * @param {unknown} data 336 336 - */ 337 337 - #cborEncode(data) { 338 338 - return encodeCBOR(data); 339 339 - } 340 340 - 341 341 - /** 342 342 - * @param {object} msg 343 343 - */ 344 344 - #send(msg) { 345 345 - if (this.#ws?.readyState === WebSocket.OPEN) { 346 346 - this.#ws.send(this.#cborEncode(msg)); 347 347 - } 348 348 - } 349 349 - 350 350 - /** 351 351 - * @param {DocumentId} documentId 352 352 - * @returns {string | undefined} 353 353 - */ 354 354 - #nameForDocId(documentId) { 355 355 - for (const name of COLLECTIONS) { 356 356 - if (this.#docIds[name] === documentId) return name; 357 357 - } 358 358 - return undefined; 359 359 - } 360 360 - } 361 361 - 362 362 - export default AutomergeRepoServerOutput; 363 363 - 364 364 - //////////////////////////////////////////// 365 365 - // REGISTER 366 366 - //////////////////////////////////////////// 367 367 - 368 368 - export const CLASS = AutomergeRepoServerOutput; 369 369 - export const NAME = "dob-automerge-repo-server"; 370 370 - 371 371 - customElements.define(NAME, AutomergeRepoServerOutput);

-3

src/index.vto

reviewed

··· 90 90 desc: "Supplies the tracks from the given output to the given search processor whenever the tracks collection changes. Additionally it can perform a search and other ways to reduce the scope of tracks based on the given scope engine. Provides a `tracks` signal similar to `output.tracks.collection`" 91 91 92 92 output: 93 93 - - title: "Bytes / Automerge Repo" 94 94 - desc: "Sync with an Automerge repo [sync server](https://github.com/automerge/automerge-repo-sync-server/tree/main)." 95 95 - todo: true 96 93 - url: "components/output/polymorphic/indexed-db/element.js" 97 94 title: "Polymorphic / IndexedDB" 98 95 desc: "Stores output into the local indexedDB. Supports any type of data that indexedDB supports."