feat: introduce automerge + remove orchestrator dependency in s3 input applet

+4

astro.config.js

··· 1 1 import { defineConfig } from "astro/config"; 2 2 import scope from "astro-scope"; 3 + import wasm from "vite-plugin-wasm"; 3 4 4 5 import purgecss from "astro-purgecss"; 5 6 ··· 7 8 integrations: [scope(), purgecss()], 8 9 build: { 9 10 inlineStylesheets: "never", 11 + }, 12 + vite: { 13 + plugins: [wasm()], 10 14 }, 11 15 });

+2

deno.lock

··· 22 22 "packageJson": { 23 23 "dependencies": [ 24 24 "npm:98.css@~0.1.21", 25 + "npm:@automerge/automerge@^2.2.9", 25 26 "npm:@jsr/bradenmacdonald__s3-lite-client@0.9", 26 27 "npm:@jsr/std__media-types@^1.1.0", 27 28 "npm:@picocss/pico@^2.1.1", ··· 41 42 "npm:spellcaster@6", 42 43 "npm:throttle-debounce@^5.0.2", 43 44 "npm:uri-js@^4.4.1", 45 + "npm:vite-plugin-wasm@^3.4.1", 44 46 "npm:webamp@^1.5.0", 45 47 "npm:xxh32@^2.0.5" 46 48 ]

+35 -1

package-lock.json

··· 5 5 "packages": { 6 6 "": { 7 7 "dependencies": { 8 + "@automerge/automerge": "^2.2.9", 8 9 "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0", 9 10 "@picocss/pico": "^2.1.1", 10 11 "@std/media-types": "npm:@jsr/std__media-types@^1.1.0", ··· 29 30 "astro-purgecss": "^5.2.2", 30 31 "astro-scope": "^3.0.1", 31 32 "purgecss": "^7.0.2", 32 - "sass": "^1.87.0" 33 + "sass": "^1.87.0", 34 + "vite-plugin-wasm": "^3.4.1" 33 35 } 34 36 }, 35 37 "node_modules/@assemblyscript/loader": { ··· 107 109 }, 108 110 "engines": { 109 111 "node": "^18.17.1 || ^20.3.0 || >=22.0.0" 112 + } 113 + }, 114 + "node_modules/@automerge/automerge": { 115 + "version": "2.2.9", 116 + "resolved": "https://registry.npmjs.org/@automerge/automerge/-/automerge-2.2.9.tgz", 117 + "integrity": "sha512-6HM52Ops79hAQBWMg/t0MNfGOdEiXyenQjO9F1hKZq0RWDsMLpPa1SzRy/C4/4UyX67sTHuA5CwBpH34SpfZlA==", 118 + "license": "MIT", 119 + "dependencies": { 120 + "uuid": "^9.0.0" 110 121 } 111 122 }, 112 123 "node_modules/@babel/helper-string-parser": { ··· 6783 6794 "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==", 6784 6795 "license": "ISC" 6785 6796 }, 6797 + "node_modules/uuid": { 6798 + "version": "9.0.1", 6799 + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", 6800 + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", 6801 + "funding": [ 6802 + "https://github.com/sponsors/broofa", 6803 + "https://github.com/sponsors/ctavan" 6804 + ], 6805 + "license": "MIT", 6806 + "bin": { 6807 + "uuid": "dist/bin/uuid" 6808 + } 6809 + }, 6786 6810 "node_modules/vfile": { 6787 6811 "version": "6.0.3", 6788 6812 "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", ··· 6897 6921 "yaml": { 6898 6922 "optional": true 6899 6923 } 6924 + } 6925 + }, 6926 + "node_modules/vite-plugin-wasm": { 6927 + "version": "3.4.1", 6928 + "resolved": "https://registry.npmjs.org/vite-plugin-wasm/-/vite-plugin-wasm-3.4.1.tgz", 6929 + "integrity": "sha512-ja3nSo2UCkVeitltJGkS3pfQHAanHv/DqGatdI39ja6McgABlpsZ5hVgl6wuR8Qx5etY3T5qgDQhOWzc5RReZA==", 6930 + "dev": true, 6931 + "license": "MIT", 6932 + "peerDependencies": { 6933 + "vite": "^2 || ^3 || ^4 || ^5 || ^6" 6900 6934 } 6901 6935 }, 6902 6936 "node_modules/vitefu": {

+3 -1

package.json

··· 1 1 { 2 2 "dependencies": { 3 + "@automerge/automerge": "^2.2.9", 3 4 "@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0", 4 5 "@picocss/pico": "^2.1.1", 5 6 "@std/media-types": "npm:@jsr/std__media-types@^1.1.0", ··· 24 25 "astro-purgecss": "^5.2.2", 25 26 "astro-scope": "^3.0.1", 26 27 "purgecss": "^7.0.2", 27 - "sass": "^1.87.0" 28 + "sass": "^1.87.0", 29 + "vite-plugin-wasm": "^3.4.1" 28 30 } 29 31 }

+4

src/pages/configurator/input/_applet.astro

··· 54 54 //////////////////////////////////////////// 55 55 // ACTIONS 56 56 //////////////////////////////////////////// 57 + const contextualize = async (tracks: Track[]) => { 58 + await input.s3.sendAction("contextualize", tracks); 59 + }; 57 60 58 61 const list = async (cachedTracks: Track[] = []) => { 59 62 const groups = cachedTracks.reduce( ··· 107 110 } 108 111 }; 109 112 113 + context.setActionHandler("contextualize", contextualize); 110 114 context.setActionHandler("list", list); 111 115 context.setActionHandler("resolve", resolve); 112 116

+8

src/pages/configurator/input/_manifest.json

··· 3 3 "title": "Diffuse Configurator | Input", 4 4 "entrypoint": "index.html", 5 5 "actions": { 6 + "contextualize": { 7 + "title": "Contextualize", 8 + "params_schema": { 9 + "type": "array", 10 + "description": "Array of tracks", 11 + "items": { "type": "object" } 12 + } 13 + }, 6 14 "list": { 7 15 "title": "List", 8 16 "description": "List tracks from all inputs.",

+9 -18

src/pages/input/s3/_applet.astro

··· 38 38 39 39 <script> 40 40 import { S3Client } from "@bradenmacdonald/s3-lite-client"; 41 - import { type AppletEvent, applets } from "@web-applets/sdk"; 41 + import { applets } from "@web-applets/sdk"; 42 42 import { computed, effect, Signal, signal } from "spellcaster"; 43 43 import { type Props, repeat, tags, text } from "spellcaster/hyperscript.js"; 44 44 import * as IDB from "idb-keyval"; 45 45 import * as URI from "uri-js"; 46 46 import QS from "query-string"; 47 47 48 - import type { Output, Track } from "@applets/core/types.d.ts"; 49 - import { applet } from "@scripts/theme"; 48 + import type { Track } from "@applets/core/types.d.ts"; 50 49 51 50 import manifest from "./_manifest.json"; 52 51 import { isAudioFile } from "@scripts/inputs/common"; ··· 89 88 // Register applet 90 89 const context = applets.register(); 91 90 92 - // Applet connections 93 - const orchestrator = { 94 - output: await applet<Output>("../../orchestrator/output-management", { 95 - context: self.top || self.parent, 96 - }), 97 - }; 98 - 99 - // Watch for data changes 100 - orchestrator.output.addEventListener("data", async (event: AppletEvent) => { 101 - await loadBuckets(); 102 - }); 103 - 104 91 //////////////////////////////////////////// 105 92 // UI 106 93 //////////////////////////////////////////// ··· 225 212 return { supported: true }; 226 213 }; 227 214 215 + const contextualize = async (tracks: Track[]) => { 216 + const b = bucketsFromTracks(tracks); 217 + setBuckets({ ...buckets(), ...b }); 218 + }; 219 + 228 220 const list = async (cachedTracks: Track[] = []) => { 229 221 const cache = cachedTracks.reduce((acc: Record<string, Track>, t: Track) => { 230 222 const uri = URI.parse(t.uri); ··· 283 275 const unmount = async () => {}; 284 276 285 277 context.setActionHandler("consult", consult); 278 + context.setActionHandler("contextualize", contextualize); 286 279 context.setActionHandler("list", list); 287 280 context.setActionHandler("resolve", resolve); 288 281 context.setActionHandler("mount", mount); ··· 341 334 342 335 async function loadBuckets() { 343 336 const i = await IDB.get(IDB_BUCKETS); 344 - const t = bucketsFromTracks(orchestrator.output.data.tracks); 345 - 346 - return { ...i, ...t }; 337 + return i; 347 338 } 348 339 349 340 function parseURI(uriString: string): Bucket | undefined {

+8

src/pages/input/s3/_manifest.json

··· 13 13 "description": "The uri to check the availability of." 14 14 } 15 15 }, 16 + "contextualize": { 17 + "title": "Contextualize", 18 + "params_schema": { 19 + "type": "array", 20 + "description": "Array of tracks", 21 + "items": { "type": "object" } 22 + } 23 + }, 16 24 "list": { 17 25 "title": "List", 18 26 "description": "List tracks.",

+18 -14

src/pages/orchestrator/input-cache/_applet.astro

··· 1 1 <script> 2 2 import { applets } from "@web-applets/sdk"; 3 3 4 - import type { Output, Track } from "@applets/core/types.d.ts"; 5 - import { applet, waitUntilAppletIsReady } from "@scripts/theme"; 4 + import type { Track } from "@applets/core/types.d.ts"; 5 + import { applet, waitUntilAppletData, waitUntilAppletIsReady } from "@scripts/theme"; 6 6 7 7 //////////////////////////////////////////// 8 8 // SETUP 9 9 //////////////////////////////////////////// 10 - const context = applets.register<{ ready: boolean }>(); 10 + import type * as OutputOrchestrator from "@applets/orchestrator/output-management/types.d.ts"; 11 + 12 + const context = applets.register<{ isProcessing: boolean; ready: boolean }>(); 11 13 const topContext = self.top || self.parent; 12 14 13 15 // Initial data 14 16 context.data = { 17 + isProcessing: false, 15 18 ready: false, 16 19 }; 17 20 ··· 21 24 }; 22 25 23 26 const orchestrator = { 24 - output: await applet<Output>("../../orchestrator/output-management", { 27 + output: await applet<OutputOrchestrator.State>("../../orchestrator/output-management", { 25 28 context: topContext, 26 29 }), 27 30 }; ··· 33 36 }; 34 37 35 38 // 🚀 36 - orchestrator.output.addEventListener( 37 - "data", 38 - () => { 39 - process(); 40 - }, 41 - { once: true }, 42 - ); 39 + waitUntilAppletData(orchestrator.output, (d) => !!d?.hasSyncedTracks).then(() => process()); 43 40 44 41 //////////////////////////////////////////// 45 42 // ACTIONS ··· 47 44 context.setActionHandler("process", process); 48 45 49 46 async function process() { 47 + if (context.data.isProcessing) return; 48 + context.data = { ...context.data, isProcessing: true }; 49 + console.log("🪵 Processing initiated"); 50 + 50 51 await waitUntilAppletIsReady(configurator.input); 51 52 52 - const cachedTracks = orchestrator.output.data.tracks; 53 + const cachedTracks = orchestrator.output.data.tracks.collection; 54 + await configurator.input.sendAction("contextualize", cachedTracks); 55 + 53 56 const tracks = await configurator.input.sendAction<Track[]>("list", cachedTracks, { 54 57 timeoutDuration: 60000 * 60 * 24, 55 58 }); ··· 99 102 timeoutDuration: 60000 * 2, 100 103 }); 101 104 102 - // Log 105 + // Fin 103 106 console.log("🪵 Processing completed"); 107 + context.data = { ...context.data, isProcessing: false }; 104 108 } 105 109 106 110 //////////////////////////////////////////// 107 111 // 🚦 108 112 //////////////////////////////////////////// 109 - context.data = { ready: true }; 113 + context.data = { ...context.data, ready: true }; 110 114 </script>

+57 -21

src/pages/orchestrator/output-management/_applet.astro

··· 2 2 <script> 3 3 import { applets } from "@web-applets/sdk"; 4 4 import { debounce } from "throttle-debounce"; 5 + import * as Automerge from "@automerge/automerge"; 5 6 6 7 import type { Track } from "@applets/core/types.d.ts"; 7 8 import type { State } from "./types.d.ts"; ··· 14 15 15 16 // Initial data 16 17 context.data = { 18 + tracks: Automerge.from({ collection: [] }, {}), 19 + 20 + hasSyncedTracks: false, 21 + 17 22 ready: false, 18 - tracks: [], 19 23 }; 20 24 21 25 // Applet connections ··· 24 28 }; 25 29 26 30 // Load tracks 27 - loadTracks().then((tracks) => { 28 - update({ tracks }); 31 + loadTracks().then((doc) => { 32 + if (doc) { 33 + const mergedDoc = Automerge.merge(doc, context.data.tracks); 34 + update({ tracks: mergedDoc }); 35 + } 36 + 37 + update({ hasSyncedTracks: true }); 29 38 }); 30 39 31 40 // State helpers ··· 33 42 context.data = { ...context.data, ...partial }; 34 43 } 35 44 45 + function updateTracks(tracks: Track[]): Automerge.Doc<{ collection: Track[] }> { 46 + const doc = Automerge.change(context.data.tracks, (d) => { 47 + d.collection = cleanUndefinedValuesForTracks(tracks); 48 + }); 49 + 50 + update({ tracks: doc }); 51 + 52 + return doc; 53 + } 54 + 36 55 //////////////////////////////////////////// 37 56 // LOADERS 38 57 //////////////////////////////////////////// 39 - async function loadTracks(): Promise<Track[]> { 58 + async function loadTracks() { 40 59 await waitUntilAppletIsReady(configurator.output); 41 60 42 61 const data = await configurator.output.sendAction( ··· 49 68 }, 50 69 ); 51 70 71 + console.log("🔮 Loading tracks, got:", data); 72 + 52 73 if (!data) { 53 - return []; 74 + return undefined; 54 75 } 55 76 56 - return decode(data as Uint8Array); 77 + return Automerge.load<{ collection: Track[] }>(data as Uint8Array); 57 78 } 58 79 59 80 //////////////////////////////////////////// 60 81 // ACTIONS 61 82 //////////////////////////////////////////// 62 83 const tracksHandler = (tracks: Track[]) => { 63 - update({ tracks }); 84 + const doc = updateTracks(tracks); 85 + 86 + console.log("🔮 Saving tracks"); 64 87 65 88 // Save tracks to output, but only the ones that need to be saved. 66 - // TODO: For each track.uri scheme ask the output configurator if it needs to be cached. 67 - saveTracksToOutput(tracks); 89 + // TODO: For each track.uri scheme ask the input configurator if it needs to be cached? 90 + saveTracksToOutput(doc); 68 91 }; 69 92 70 - const saveTracksToOutput = debounce(5000, async function (tracks: Track[]) { 71 - const data = encode(tracks); 93 + const saveTracksToOutput = debounce( 94 + 5000, 95 + async function (doc: Automerge.Doc<{ collection: Track[] }>) { 96 + const data = Automerge.save(doc); 72 97 73 - await configurator.output.sendAction("put", { 74 - name: "tracks.json", 75 - data, 76 - }); 77 - }); 98 + await configurator.output.sendAction("put", { 99 + name: "tracks.json", 100 + data, 101 + }); 102 + }, 103 + ); 78 104 79 105 context.setActionHandler("tracks", tracksHandler); 80 106 81 107 //////////////////////////////////////////// 82 108 // 🛠️ 83 109 //////////////////////////////////////////// 84 - function decode(data: Uint8Array) { 85 - return JSON.parse(new TextDecoder().decode(data)); 86 - } 110 + function cleanUndefinedValuesForTracks(tracks: Track[]): Track[] { 111 + return tracks.map((track) => { 112 + const t = { ...track }; 113 + 114 + if (t.tags) { 115 + if ("album" in t.tags && t.tags.album === undefined) delete t.tags.album; 116 + if ("artist" in t.tags && t.tags.artist === undefined) delete t.tags.artist; 117 + if ("genre" in t.tags && t.tags.genre === undefined) delete t.tags.genre; 118 + if ("year" in t.tags && t.tags.year === undefined) delete t.tags.year; 119 + 120 + if ("of" in t.tags.disc && t.tags.disc.of === undefined) delete t.tags.disc.of; 121 + if ("of" in t.tags.track && t.tags.track.of === undefined) delete t.tags.track.of; 122 + } 87 123 88 - function encode(data: Object) { 89 - return new TextEncoder().encode(JSON.stringify(data)); 124 + return t; 125 + }); 90 126 } 91 127 92 128 ////////////////////////////////////////////

+9 -2

src/pages/orchestrator/output-management/types.d.ts

··· 1 - import { Output } from "@applets/core/types"; 1 + import type { Doc } from "@automerge/automerge"; 2 + import type { Output } from "@applets/core/types"; 3 + 4 + export type State = { 5 + tracks: Doc<{ collection: Output["tracks"] }>; 6 + 7 + hasSyncedTracks: boolean; 2 8 3 - export type State = Output & { ready: boolean }; 9 + ready: boolean; 10 + };

+3 -2

src/pages/orchestrator/single-queue/_applet.astro

··· 9 9 //////////////////////////////////////////// 10 10 import type * as AudioEngine from "@applets/engine/audio/types.d.ts"; 11 11 import type * as QueueEngine from "@applets/engine/queue/types.d.ts"; 12 + import type * as OutputOrchestrator from "@applets/orchestrator/output-management/types.d.ts"; 12 13 13 14 // Register applet 14 15 const context = applets.register<unknown>(); ··· 24 25 }; 25 26 26 27 const orchestrator = { 27 - output: await applet<Output>("../../orchestrator/output-management", { 28 + output: await applet<OutputOrchestrator.State>("../../orchestrator/output-management", { 28 29 context: self.top || self.parent, 29 30 }), 30 31 }; ··· 39 40 // into a usable audio URL. 40 41 engine.queue.sendAction( 41 42 "add", 42 - orchestrator.output.data.tracks.map((track: Track) => { 43 + orchestrator.output.data.tracks.collection.map((track: Track) => { 43 44 return { 44 45 expiresAt: Infinity, 45 46 id: track.id,

+10 -3

src/scripts/theme.ts

··· 132 132 return test !== Object(test); 133 133 } 134 134 135 - export function waitUntilAppletIsReady(applet: Applet): Promise<void> { 135 + export function waitUntilAppletData<A>( 136 + applet: Applet<A>, 137 + dataFn: (a: A | undefined) => boolean, 138 + ): Promise<void> { 136 139 return new Promise((resolve) => { 137 - if (applet.data?.ready === true) { 140 + if (dataFn(applet.data) === true) { 138 141 resolve(); 139 142 return; 140 143 } 141 144 142 145 const callback = (event: AppletEvent) => { 143 - if (event.data?.ready === true) { 146 + if (dataFn(event.data) === true) { 144 147 applet.removeEventListener("data", callback); 145 148 resolve(); 146 149 } ··· 149 152 applet.addEventListener("data", callback); 150 153 }); 151 154 } 155 + 156 + export function waitUntilAppletIsReady(applet: Applet): Promise<void> { 157 + return waitUntilAppletData(applet, (data) => !!data?.ready); 158 + }

+3 -2

src/scripts/themes/webamp/index.ts

··· 12 12 //////////////////////////////////////////// 13 13 // 🗂️ Applets 14 14 //////////////////////////////////////////// 15 + import type * as OutputOrchestrator from "@applets/orchestrator/output-management/types.d.ts"; 15 16 16 17 const configurator = { 17 18 input: await applet("../../configurator/input"), 18 19 }; 19 20 20 21 const orchestrator = { 21 - output: await applet<Output>("../../orchestrator/output-management"), 22 + output: await applet<OutputOrchestrator.State>("../../orchestrator/output-management"), 22 23 23 24 // TODO: Should this be explicitely be ran after the output orchestrator is loaded? 24 25 input: await applet("../../orchestrator/input-cache"), ··· 47 48 // 🛠️ 48 49 //////////////////////////////////////////// 49 50 async function loadTracks(): Promise<URLTrack[]> { 50 - return await orchestrator.output.data.tracks.reduce( 51 + return await orchestrator.output.data.tracks.collection.reduce( 51 52 async (promise: Promise<URLTrack[]>, track: Track) => { 52 53 const acc = await promise; 53 54

Configure Feed

Configure Feed