Experiment to rebuild Diffuse using web applets.
feat: improve metadata processing, don't download the entire track
+173
-41
+2
deno.lock
+2
deno.lock
···
24
24
"npm:@jsr/bradenmacdonald__s3-lite-client@0.9",
25
25
"npm:@jsr/std__media-types@^1.1.0",
26
26
"npm:@picocss/pico@^2.1.1",
27
+
"npm:@tokenizer/http@~0.9.2",
28
+
"npm:@tokenizer/range@0.13",
27
29
"npm:@types/throttle-debounce@^5.0.2",
28
30
"npm:astro-purgecss@^5.2.2",
29
31
"npm:astro-scope@^3.0.1",
+81
-7
package-lock.json
+81
-7
package-lock.json
···
8
8
"@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0",
9
9
"@picocss/pico": "^2.1.1",
10
10
"@std/media-types": "npm:@jsr/std__media-types@^1.1.0",
11
+
"@tokenizer/http": "^0.9.2",
12
+
"@tokenizer/range": "^0.13.0",
11
13
"@web-applets/sdk": "https://gitpkg.vercel.app/unternet-co/web-applets/sdk?tokono.ma/experiment&scripts.postinstall=npm%20i%20%40types%2Fnode%20%26%26%20npx%20tsc",
12
14
"iconoir": "^7.11.0",
13
15
"idb-keyval": "^6.2.1",
14
16
"music-metadata": "^11.2.3",
15
17
"native-file-system-adapter": "^3.0.1",
16
-
"node-s3-url-encode": "^0.0.4",
17
18
"query-string": "^9.1.2",
18
19
"spellcaster": "^6.0.0",
19
20
"throttle-debounce": "^5.0.2",
···
1703
1704
"tslib": "^2.8.0"
1704
1705
}
1705
1706
},
1707
+
"node_modules/@tokenizer/http": {
1708
+
"version": "0.9.2",
1709
+
"resolved": "https://registry.npmjs.org/@tokenizer/http/-/http-0.9.2.tgz",
1710
+
"integrity": "sha512-rzJwHcqDjO3FdBPr+FK2R6dYE6Qbg6QZP7S47rhCEtG+/YqEFLqZ+gFCLcL8y5D39aYQB9vDssiwbsJlRLePPg==",
1711
+
"license": "MIT",
1712
+
"dependencies": {
1713
+
"@tokenizer/range": "^0.12.0",
1714
+
"debug": "^4.3.7",
1715
+
"strtok3": "^10.0.0"
1716
+
},
1717
+
"funding": {
1718
+
"type": "github",
1719
+
"url": "https://github.com/sponsors/Borewit"
1720
+
}
1721
+
},
1722
+
"node_modules/@tokenizer/http/node_modules/@tokenizer/range": {
1723
+
"version": "0.12.0",
1724
+
"resolved": "https://registry.npmjs.org/@tokenizer/range/-/range-0.12.0.tgz",
1725
+
"integrity": "sha512-xvJ1OflWjopkC5EgLge+9HrwsWStgVewQkmusoF2BxgCuGdm1KuhZAMVMNzC7h1WNei9JA6xKQlkbPNJtjZ6aw==",
1726
+
"license": "MIT",
1727
+
"dependencies": {
1728
+
"debug": "^4.3.7",
1729
+
"strtok3": "^9.1.1"
1730
+
},
1731
+
"engines": {
1732
+
"node": ">=16"
1733
+
},
1734
+
"funding": {
1735
+
"type": "github",
1736
+
"url": "https://github.com/sponsors/Borewit"
1737
+
}
1738
+
},
1739
+
"node_modules/@tokenizer/http/node_modules/@tokenizer/range/node_modules/strtok3": {
1740
+
"version": "9.1.1",
1741
+
"resolved": "https://registry.npmjs.org/strtok3/-/strtok3-9.1.1.tgz",
1742
+
"integrity": "sha512-FhwotcEqjr241ZbjFzjlIYg6c5/L/s4yBGWSMvJ9UoExiSqL+FnFA/CaeZx17WGaZMS/4SOZp8wH18jSS4R4lw==",
1743
+
"license": "MIT",
1744
+
"dependencies": {
1745
+
"@tokenizer/token": "^0.3.0",
1746
+
"peek-readable": "^5.3.1"
1747
+
},
1748
+
"engines": {
1749
+
"node": ">=16"
1750
+
},
1751
+
"funding": {
1752
+
"type": "github",
1753
+
"url": "https://github.com/sponsors/Borewit"
1754
+
}
1755
+
},
1756
+
"node_modules/@tokenizer/http/node_modules/peek-readable": {
1757
+
"version": "5.4.2",
1758
+
"resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-5.4.2.tgz",
1759
+
"integrity": "sha512-peBp3qZyuS6cNIJ2akRNG1uo1WJ1d0wTxg/fxMdZ0BqCVhx242bSFHM9eNqflfJVS9SsgkzgT/1UgnsurBOTMg==",
1760
+
"license": "MIT",
1761
+
"engines": {
1762
+
"node": ">=14.16"
1763
+
},
1764
+
"funding": {
1765
+
"type": "github",
1766
+
"url": "https://github.com/sponsors/Borewit"
1767
+
}
1768
+
},
1706
1769
"node_modules/@tokenizer/inflate": {
1707
1770
"version": "0.2.7",
1708
1771
"resolved": "https://registry.npmjs.org/@tokenizer/inflate/-/inflate-0.2.7.tgz",
···
1715
1778
},
1716
1779
"engines": {
1717
1780
"node": ">=18"
1781
+
},
1782
+
"funding": {
1783
+
"type": "github",
1784
+
"url": "https://github.com/sponsors/Borewit"
1785
+
}
1786
+
},
1787
+
"node_modules/@tokenizer/range": {
1788
+
"version": "0.13.0",
1789
+
"resolved": "https://registry.npmjs.org/@tokenizer/range/-/range-0.13.0.tgz",
1790
+
"integrity": "sha512-ibLGQRU8an1g/y952+OxeZDGIj+W1HW8AQPtk26VIFWzy3tvQImmGBwYbpHJXMMAz1nhCPAAepCRptGKB8YrKg==",
1791
+
"license": "MIT",
1792
+
"dependencies": {
1793
+
"debug": "^4.4.0",
1794
+
"strtok3": "^10.2.0"
1795
+
},
1796
+
"engines": {
1797
+
"node": ">=16"
1718
1798
},
1719
1799
"funding": {
1720
1800
"type": "github",
···
5016
5096
"resolved": "https://registry.npmjs.org/node-mock-http/-/node-mock-http-1.0.0.tgz",
5017
5097
"integrity": "sha512-0uGYQ1WQL1M5kKvGRXWQ3uZCHtLTO8hln3oBjIusM75WoesZ909uQJs/Hb946i2SS+Gsrhkaa6iAO17jRIv6DQ==",
5018
5098
"dev": true
5019
-
},
5020
-
"node_modules/node-s3-url-encode": {
5021
-
"version": "0.0.4",
5022
-
"resolved": "https://registry.npmjs.org/node-s3-url-encode/-/node-s3-url-encode-0.0.4.tgz",
5023
-
"integrity": "sha512-l0IizfnxE1hb9dadzYBpA27syfL9LFkPzCKH6YWrssv2sPLjVuCent67A8GPe4isdj4bEsbgdPWLTcV4gxEg9w==",
5024
-
"license": "MIT"
5025
5099
},
5026
5100
"node_modules/normalize-path": {
5027
5101
"version": "3.0.0",
+2
package.json
+2
package.json
···
3
3
"@bradenmacdonald/s3-lite-client": "npm:@jsr/bradenmacdonald__s3-lite-client@^0.9.0",
4
4
"@picocss/pico": "^2.1.1",
5
5
"@std/media-types": "npm:@jsr/std__media-types@^1.1.0",
6
+
"@tokenizer/http": "^0.9.2",
7
+
"@tokenizer/range": "^0.13.0",
6
8
"@web-applets/sdk": "https://gitpkg.vercel.app/unternet-co/web-applets/sdk?tokono.ma/experiment&scripts.postinstall=npm%20i%20%40types%2Fnode%20%26%26%20npx%20tsc",
7
9
"iconoir": "^7.11.0",
8
10
"idb-keyval": "^6.2.1",
+1
-4
src/pages/configurator/input/_applet.astro
+1
-4
src/pages/configurator/input/_applet.astro
+4
src/pages/core/types.d.ts
+4
src/pages/core/types.d.ts
+23
-17
src/pages/orchestrator/input-cache/_applet.astro
+23
-17
src/pages/orchestrator/input-cache/_applet.astro
···
8
8
// SETUP
9
9
////////////////////////////////////////////
10
10
const context = applets.register<{ ready: boolean }>();
11
+
const topContext = self.top || self.parent;
11
12
12
13
// Initial data
13
14
context.data = {
···
16
17
17
18
// Applet connections
18
19
const configurator = {
19
-
input: await applet("../../configurator/input", { context: self.top || self.parent }),
20
+
input: await applet("../../configurator/input", { context: topContext }),
20
21
};
21
22
22
23
const orchestrator = {
23
24
output: await applet<Output>("../../orchestrator/output-management", {
24
-
context: self.parent,
25
+
context: topContext,
25
26
}),
26
27
};
27
28
28
29
const processor = {
29
-
metadataFetcher: await applet("../../processor/metadata-fetcher", { context: self.parent }),
30
+
metadataFetcher: await applet("../../processor/metadata-fetcher", {
31
+
context: topContext,
32
+
}),
30
33
};
31
34
32
35
// 🚀
···
50
53
async (promise: Promise<Track[]>, track: Track) => {
51
54
const acc = await promise;
52
55
53
-
if (track.tags) return [...acc, track];
56
+
if (track.tags && track.stats) return [...acc, track];
54
57
55
58
const getURL = await configurator.input.sendAction<string | undefined>(
56
59
"resolve",
···
60
63
},
61
64
);
62
65
63
-
if (!getURL) return acc;
66
+
const headURL = await configurator.input.sendAction<string | undefined>(
67
+
"resolve",
68
+
{ method: "HEAD", uri: track.uri },
69
+
{
70
+
timeoutDuration: 60000,
71
+
},
72
+
);
64
73
65
-
// TODO: Do we need to pass the HEAD URL too?
66
-
const meta = await processor.metadataFetcher.sendAction("extract", getURL, {
67
-
timeoutDuration: 60000,
68
-
});
74
+
if (!getURL) return acc;
69
75
70
-
const stats: TrackStats = {
71
-
duration: meta.format.duration,
72
-
};
76
+
const { stats, tags } = await processor.metadataFetcher.sendAction(
77
+
"extract",
78
+
{ urls: { get: getURL, head: headURL || getURL } },
79
+
{
80
+
timeoutDuration: 60000,
81
+
},
82
+
);
73
83
74
-
const tags: TrackTags = {
75
-
album: meta.common.album,
76
-
artist: meta.common.artist,
77
-
title: meta.common.title,
78
-
};
84
+
console.log(stats, tags);
79
85
80
86
return [...acc, { ...track, stats, tags }];
81
87
},
+3
-1
src/pages/orchestrator/single-queue/_applet.astro
+3
-1
src/pages/orchestrator/single-queue/_applet.astro
···
24
24
};
25
25
26
26
const orchestrator = {
27
-
output: await applet<Output>("../../orchestrator/output-management", { context: self.parent }),
27
+
output: await applet<Output>("../../orchestrator/output-management", {
28
+
context: self.top || self.parent,
29
+
}),
28
30
};
29
31
30
32
////////////////////////////////////////////
+45
-10
src/pages/processor/metadata-fetcher/_applet.astro
+45
-10
src/pages/processor/metadata-fetcher/_applet.astro
···
1
1
<script>
2
2
import { applets } from "@web-applets/sdk";
3
-
import { parseWebStream } from "music-metadata";
3
+
import { parseFromTokenizer } from "music-metadata";
4
4
import { contentType } from "@std/media-types";
5
5
import * as URI from "uri-js";
6
+
import * as HTTP_TOKENIZER from "@tokenizer/http";
7
+
import * as RANGE_TOKENIZER from "@tokenizer/range";
8
+
9
+
import { TrackStats, TrackTags } from "@applets/core/types";
6
10
7
11
////////////////////////////////////////////
8
12
// SETUP
9
13
////////////////////////////////////////////
10
14
const context = applets.register();
11
15
16
+
type Extraction = { stats: TrackStats; tags: TrackTags };
17
+
type Urls = { get: string; head: string };
18
+
12
19
////////////////////////////////////////////
13
20
// ACTIONS
14
21
////////////////////////////////////////////
15
22
context.setActionHandler("extract", extract);
16
23
17
-
async function extract(url: string) {
18
-
const uri = URI.parse(url);
24
+
async function extract({ urls }: { urls: Urls }) {
25
+
// Construct records
26
+
// TODO: Use other metadata lib as fallback: https://github.com/buzz/mediainfo.js
27
+
const { stats, tags } = await musicMetadataTags(urls, false);
28
+
29
+
// Fin
30
+
return { stats, tags };
31
+
}
32
+
33
+
// 🛠️
34
+
async function musicMetadataTags(urls: Urls, covers: boolean = false): Promise<Extraction> {
35
+
const httpClient = new HTTP_TOKENIZER.HttpClient(urls.head, { resolveUrl: false });
36
+
httpClient.resolvedUrl = urls.get;
37
+
38
+
const tokenizer = await RANGE_TOKENIZER.tokenizer(httpClient);
39
+
const meta = await parseFromTokenizer(tokenizer, { skipCovers: covers });
40
+
41
+
const uri = URI.parse(urls.get);
19
42
const pathParts = uri.path?.split("/");
20
-
const mimeType = pathParts?.[pathParts.length - 1]?.includes(".")
21
-
? contentType(pathParts[pathParts.length - 1].split(".").reverse()[0])
22
-
: undefined;
23
-
const resp = await fetch(url);
24
-
const stream = resp.body;
25
-
const metadata = await parseWebStream(stream, { mimeType });
43
+
const filename = pathParts?.[pathParts.length - 1];
44
+
45
+
const stats: TrackStats = {
46
+
duration: meta.format.duration,
47
+
};
48
+
49
+
const tags: TrackTags = {
50
+
album: meta.common.album,
51
+
artist: meta.common.artist,
52
+
disc: { no: meta.common.disk.no || 1, of: meta.common.disk.of ?? undefined },
53
+
genre: Array.isArray(meta.common.genre) ? meta.common.genre[0] : meta.common.genre,
54
+
title: meta.common.title || filename || urls.head,
55
+
track: { no: meta.common.track.no || 1, of: meta.common.track.of ?? undefined },
56
+
year: meta.common.year,
57
+
};
26
58
27
-
return metadata;
59
+
return {
60
+
stats,
61
+
tags,
62
+
};
28
63
}
29
64
</script>
+12
-2
src/pages/processor/metadata-fetcher/_manifest.json
+12
-2
src/pages/processor/metadata-fetcher/_manifest.json
···
7
7
"title": "Extract",
8
8
"description": "Get the metadata for a given URL.",
9
9
"params_schema": {
10
-
"type": "string",
11
-
"description": "URL"
10
+
"type": "object",
11
+
"properties": {
12
+
"urls": {
13
+
"type": "object",
14
+
"properties": {
15
+
"get": { "type": "string" },
16
+
"head": { "type": "string" }
17
+
},
18
+
"required": ["get", "head"]
19
+
}
20
+
},
21
+
"required": ["urls"]
12
22
}
13
23
}
14
24
}