WIP. A little custom music server
refactor: restructure file-parser and sync-library to improve file handling and service integration
+295
-100
+29
-76
backend/src/file-parser.ts
+29
-76
backend/src/file-parser.ts
···
1
-
import * as flac from "./flac";
2
-
import { Effect, Option, Console, Duration, Data, Stream } from "effect";
1
+
import { Effect, Console, Data, Stream } from "effect";
3
2
import { FileSystem, Path } from "@effect/platform";
3
+
import { FlacService } from "./flac/service";
4
4
5
5
const SUPPORTED_EXTENSIONS = ["flac"] as const;
6
6
type SupportedExtension = (typeof SUPPORTED_EXTENSIONS)[number];
···
10
10
message: string;
11
11
}> {}
12
12
13
-
const supportedExtensions = [".flac"] as const;
14
-
15
-
export function parseFile(path: string) {
16
-
return Effect.gen(function* () {
17
-
const extension = path.split(".").pop();
18
-
const assumedType =
19
-
extension && SUPPORTED_EXTENSIONS.includes(extension as SupportedExtension)
20
-
? (extension as SupportedExtension)
21
-
: null;
22
-
23
-
if (!assumedType) {
24
-
return yield* Effect.fail(new UnsupportedFileError({ message: "File is unsupported" }));
25
-
}
26
-
27
-
if (assumedType === "flac") {
28
-
const isFlac = yield* flac.isFlac(path);
29
-
if (isFlac) {
30
-
const metadata = yield* flac.readMetadata(path);
31
-
32
-
return metadata;
33
-
}
34
-
}
35
-
return yield* Effect.fail(new UnsupportedFileError({ message: "File is unsupported" }));
36
-
}).pipe(
37
-
Effect.tapError((e) => {
38
-
if (e._tag === "UnsupportedFileError") return Effect.succeed(null);
39
-
40
-
return Console.error(e);
41
-
}),
42
-
Effect.withSpan("parseFile"),
43
-
);
44
-
}
45
-
46
-
export function parseManyFiles(paths: string[]) {
47
-
return Effect.gen(function* () {
48
-
const tasks = paths.map((path) => Effect.option(parseFile(path)));
49
-
50
-
const files = yield* Effect.all(tasks, {
51
-
concurrency: 10,
52
-
});
53
-
54
-
const successful = files.filter(Option.isSome).map((file) => file.value);
55
-
56
-
return successful;
57
-
}).pipe(Effect.withSpan("parseManyFiles"));
58
-
}
59
-
60
-
export function readDirectory(dirPath: string, skip: string[] = []) {
61
-
return Effect.gen(function* () {
62
-
const fs = yield* FileSystem.FileSystem;
63
-
const path = yield* Path.Path;
64
-
65
-
const files = yield* fs.readDirectory(dirPath, {
66
-
recursive: true,
67
-
});
68
-
69
-
yield* Console.log(files);
70
-
71
-
const relative = files.map((file) => path.resolve(dirPath, file));
72
-
yield* Console.log(relative);
73
-
74
-
const filtered = relative.filter((file) => skip.includes(file) === false);
75
-
yield* Console.log("FILTERED", filtered);
76
-
77
-
if (filtered.length === 0) {
78
-
return yield* Effect.succeed([]);
79
-
}
80
-
81
-
return yield* parseManyFiles(filtered);
82
-
}).pipe(Effect.withSpan("readDirectory"));
83
-
}
84
-
85
-
export const readDirectoryStream = Effect.fn("read-directory-stream")(function* (dirPath: string, skip: string[] = []) {
13
+
export const readDirectory = Effect.fn("read-directory")(function* (dirPath: string, skip: string[] = []) {
86
14
const fs = yield* FileSystem.FileSystem;
87
15
const path = yield* Path.Path;
88
16
···
94
22
95
23
const stream = Stream.fromIterable(files).pipe(
96
24
Stream.map((file) => path.resolve(dirPath, file)),
97
-
Stream.filter((file) => supportedExtensions.some((ext) => file.endsWith(ext)) === true),
25
+
Stream.filter((file) => SUPPORTED_EXTENSIONS.some((ext) => file.endsWith(ext)) === true),
98
26
Stream.filter((file) => skip.includes(file) === false),
99
27
Stream.mapEffect((file) => parseFile(file), {
100
28
concurrency: 10,
···
104
32
105
33
return stream;
106
34
});
35
+
36
+
export const parseFile = Effect.fn("parse-file")(function* (path: string) {
37
+
const extension = path.split(".").pop();
38
+
const assumedType =
39
+
extension && SUPPORTED_EXTENSIONS.includes(extension as SupportedExtension)
40
+
? (extension as SupportedExtension)
41
+
: null;
42
+
43
+
if (!assumedType) {
44
+
return yield* Effect.fail(new UnsupportedFileError({ message: "File is unsupported" }));
45
+
}
46
+
47
+
if (assumedType === "flac") {
48
+
49
+
const flacService = yield* FlacService;
50
+
51
+
const isFlac = yield* flacService.isFlac(path);
52
+
if (isFlac) {
53
+
const metadata = yield* flacService.readMetadata(path);
54
+
55
+
return metadata;
56
+
}
57
+
}
58
+
return yield* Effect.fail(new UnsupportedFileError({ message: "File is unsupported" }));
59
+
});
+3
backend/src/flac/errors.ts
+3
backend/src/flac/errors.ts
+96
backend/src/flac/service.ts
+96
backend/src/flac/service.ts
···
1
+
import { Effect, Schema } from "effect";
2
+
import { FileSystem } from "@effect/platform";
3
+
import { FlacHeaderFromUint8Array, MetadataFromUint8Array } from "./transformers";
4
+
import { FlacError } from "./errors";
5
+
import { MetadataWithFilepathSchema } from "~/metadata";
6
+
7
+
const VORBIS_STREAMINFO = 4;
8
+
9
+
export class FlacService extends Effect.Service<FlacService>()("FlacService", {
10
+
effect: Effect.gen(function* () {
11
+
const fs = yield* FileSystem.FileSystem;
12
+
13
+
const isFlac = Effect.fn("isFlac")(function* (path: string) {
14
+
const file = yield* fs.readFile(path);
15
+
const slice = file.slice(0, 4).toString();
16
+
return slice === "fLaC";
17
+
});
18
+
19
+
const readMetadata = Effect.fn("flac-readMetadata")(function* (path: string) {
20
+
const fileIsFlac = yield* isFlac(path);
21
+
if (!fileIsFlac) {
22
+
return yield* Effect.fail(
23
+
new FlacError({
24
+
message: "The file you are trying to parse as FLAC is NOT FLAC",
25
+
})
26
+
);
27
+
}
28
+
29
+
const file = yield* fs.readFile(path);
30
+
31
+
let offset = 4;
32
+
let header = yield* readHeader(file, offset);
33
+
34
+
while (!header.isLast && header.streamInfo !== VORBIS_STREAMINFO) {
35
+
offset = offset + 4 + header.length;
36
+
header = yield* readHeader(file, offset);
37
+
}
38
+
39
+
offset += 4;
40
+
41
+
const vorbisComment = yield* readVorbisComment(file, offset, header.length);
42
+
43
+
const result = MetadataWithFilepathSchema.make({
44
+
...vorbisComment,
45
+
filePath: path,
46
+
});
47
+
48
+
return result;
49
+
});
50
+
51
+
return {
52
+
isFlac,
53
+
readMetadata,
54
+
};
55
+
}),
56
+
}) {}
57
+
58
+
59
+
60
+
// Helpers
61
+
const readHeader = Effect.fn("flac-readHeader")(function* (file: Uint8Array, offset: number) {
62
+
const result = yield* Schema.decode(FlacHeaderFromUint8Array)({
63
+
uint8Array: file,
64
+
offset: offset,
65
+
}).pipe(
66
+
Effect.mapError(
67
+
(e) =>
68
+
new FlacError({
69
+
cause: e,
70
+
message: "Failed reading header",
71
+
})
72
+
)
73
+
);
74
+
75
+
return result;
76
+
});
77
+
78
+
const readVorbisComment = Effect.fn("readVorbisComment")(function* (file: Uint8Array, offset: number, length: number) {
79
+
const slice = file.slice(offset, offset + length);
80
+
81
+
const vorbisComment = yield* Schema.decode(MetadataFromUint8Array)({
82
+
uint8Array: slice,
83
+
offset,
84
+
length,
85
+
}).pipe(
86
+
Effect.mapError(
87
+
(e) =>
88
+
new FlacError({
89
+
cause: e,
90
+
message: "Failed to parse Vorbis Comment",
91
+
})
92
+
)
93
+
);
94
+
95
+
return vorbisComment;
96
+
});
+138
backend/src/flac/transformers.ts
+138
backend/src/flac/transformers.ts
···
1
+
import { Effect, Option, ParseResult, Schema } from "effect";
2
+
3
+
import { type Metadata, MetadataSchema } from "~/metadata";
4
+
import { Bit, Int24, safeParseInt, Uint7 } from "~/utils";
5
+
6
+
const FlacHeader = Schema.Struct({
7
+
isLast: Bit, // 1 bit
8
+
streamInfo: Uint7, // 7 bits
9
+
length: Int24, // 3 bytes
10
+
});
11
+
12
+
const FlacHeaderInput = Schema.Struct({
13
+
uint8Array: Schema.Uint8ArrayFromSelf,
14
+
offset: Schema.Number,
15
+
});
16
+
17
+
export const FlacHeaderFromUint8Array = Schema.transformOrFail(
18
+
FlacHeaderInput,
19
+
FlacHeader,
20
+
{
21
+
strict: true,
22
+
decode({ uint8Array, offset }, _, ast) {
23
+
return Effect.gen(function* () {
24
+
if (uint8Array.length < 4) {
25
+
return yield* ParseResult.fail(new ParseResult.Type(ast, uint8Array, "UIntArray is too short"));
26
+
}
27
+
28
+
const dataView = new DataView(uint8Array.buffer, offset, 4);
29
+
30
+
const header = {
31
+
isLast: ((dataView.getUint8(0) & 0x80) === 0x80 ? 1 : 0) as Bit, // 1 bit
32
+
streamInfo: dataView.getUint8(0) & 0x7f, // 7 bits
33
+
length: dataView.getUint32(0, false) & 0xffffff, // 3 byte
34
+
};
35
+
36
+
return yield* ParseResult.succeed(header);
37
+
});
38
+
},
39
+
encode(x, _, ast) {
40
+
return ParseResult.fail(new ParseResult.Type(ast, x, "Unimplemented"));
41
+
},
42
+
},
43
+
);
44
+
45
+
46
+
const MetadataInput = Schema.Struct({
47
+
uint8Array: Schema.Uint8ArrayFromSelf,
48
+
offset: Schema.Number,
49
+
length: Schema.Number,
50
+
});
51
+
52
+
export const MetadataFromUint8Array = Schema.transformOrFail(
53
+
MetadataInput,
54
+
MetadataSchema,
55
+
{
56
+
strict: true,
57
+
decode({ uint8Array, offset, length }, _, ast) {
58
+
return Effect.gen(function* () {
59
+
const dv = new DataView(uint8Array.buffer, offset, length);
60
+
61
+
let cursor = 0;
62
+
63
+
const vendorStringLength = dv.getUint32(cursor, true);
64
+
cursor += 4;
65
+
66
+
cursor += vendorStringLength;
67
+
68
+
const numberOfFields = dv.getUint32(cursor, true);
69
+
cursor += 4;
70
+
71
+
const metadata: Partial<Metadata & { artists: string[] }> = {};
72
+
73
+
for (let i = 0; i < numberOfFields; i++) {
74
+
const fieldLength = dv.getUint32(cursor, true);
75
+
cursor += 4;
76
+
77
+
const fieldValue = uint8Array.slice(cursor, cursor + fieldLength).toString();
78
+
cursor += fieldLength;
79
+
80
+
const parsedField = parseFieldValue(fieldValue);
81
+
82
+
if (!parsedField) {
83
+
continue;
84
+
}
85
+
86
+
const { key, value } = parsedField;
87
+
88
+
switch (key.toUpperCase()) {
89
+
case "ALBUM":
90
+
metadata.album = value;
91
+
break;
92
+
case "ARTIST":
93
+
if (!metadata.artists) {
94
+
metadata.artists = [];
95
+
}
96
+
metadata.artists.push(value);
97
+
break;
98
+
case "ALBUM ARTIST":
99
+
metadata.albumArtist = value;
100
+
break;
101
+
case "TITLE":
102
+
metadata.title = value;
103
+
break;
104
+
case "TRACKNUMBER":
105
+
const parsedNumber = yield* safeParseInt(value, 10);
106
+
if (Option.isSome(parsedNumber)) {
107
+
metadata.trackNumber = parsedNumber.value;
108
+
}
109
+
break;
110
+
default:
111
+
continue;
112
+
}
113
+
}
114
+
115
+
metadata.albumArtist ??= metadata.artists?.at(0);
116
+
117
+
const valid = Schema.decodeUnknownOption(MetadataSchema)(metadata);
118
+
119
+
return yield* Option.match(valid, {
120
+
onSome: (value) => ParseResult.succeed(value),
121
+
onNone: () => ParseResult.fail(new ParseResult.Type(ast, metadata, "Metadata is not valid")),
122
+
});
123
+
});
124
+
},
125
+
encode(x, _, ast) {
126
+
return ParseResult.fail(new ParseResult.Type(ast, x, "Unimplemented"));
127
+
},
128
+
},
129
+
);
130
+
131
+
132
+
function parseFieldValue(str: string): { key: string; value: string } | null {
133
+
const [key, value] = str.split("=").map((x) => x.trim());
134
+
if (!key || !value) {
135
+
return null;
136
+
}
137
+
return { key: key.toUpperCase(), value };
138
+
}
+12
-7
backend/src/index.ts
+12
-7
backend/src/index.ts
···
1
-
import { BunContext, BunRuntime } from "@effect/platform-bun";
2
-
import { Effect, Layer } from "effect";
1
+
import { BunContext, BunFileSystem, BunRuntime } from "@effect/platform-bun";
2
+
import { Config, Effect, Layer } from "effect";
3
3
import { DatabaseLive } from "./db";
4
-
import { Env, EnvLive } from "./env";
5
4
import { syncLibraryStream } from "./sync-library";
6
5
import { OtelLive } from "./otel";
7
6
import { startApi } from "./api";
7
+
import { FlacService } from "./flac/service";
8
8
9
-
const layers = Layer.mergeAll(BunContext.layer, EnvLive, DatabaseLive.Default);
9
+
const layers = Layer.mergeAll(
10
+
BunContext.layer,
11
+
OtelLive,
12
+
DatabaseLive.Default,
13
+
FlacService.Default.pipe(Layer.provide(BunFileSystem.layer)),
14
+
);
10
15
11
16
const main = Effect.gen(function* () {
12
-
const env = yield* Env.pipe(Effect.flatMap((x) => x.getEnv)).pipe(Effect.withSpan("env"));
17
+
const folderPath = yield* Config.string("FOLDER_PATH");
13
18
14
-
yield* syncLibraryStream(env.FOLDER_PATH);
19
+
yield* syncLibraryStream(folderPath);
15
20
16
21
startApi();
17
-
}).pipe(Effect.provide(OtelLive), Effect.provide(layers));
22
+
}).pipe(Effect.provide(layers));
18
23
19
24
BunRuntime.runMain(main);
+17
-17
backend/src/sync-library.ts
+17
-17
backend/src/sync-library.ts
···
1
1
import { Effect, Stream, Chunk, Console } from "effect";
2
2
import { DatabaseLive } from "./db";
3
-
import { readDirectoryStream } from "./file-parser";
3
+
import { readDirectory } from "./file-parser";
4
4
5
5
import { albumTable, artistTable, artistToAlbumTable, fileTable, songTable, songToArtistTable } from "./db/schema";
6
6
import type { MetadataWithFilepathSchema } from "./metadata";
7
7
import { colors } from "./chalk";
8
8
9
+
type Metadata = typeof MetadataWithFilepathSchema.Type;
10
+
9
11
export const syncLibraryStream = Effect.fn("sync-library-stream")(function* (libraryPath: string) {
10
12
const db = yield* DatabaseLive;
11
13
12
14
//TODO: reaindex old files / updated files
13
15
const alreadyIndexed = yield* db.query.fileTable.findMany();
14
16
15
-
const stream = yield* readDirectoryStream(
17
+
const stream = yield* readDirectory(
16
18
libraryPath,
17
-
alreadyIndexed.map((x) => x.path),
19
+
alreadyIndexed.map((x) => x.path)
18
20
);
19
21
20
22
yield* stream.pipe(
···
27
29
"by",
28
30
colors.FgYellow,
29
31
x.artists.join(","),
30
-
colors.Reset,
31
-
),
32
+
colors.Reset
33
+
)
32
34
),
33
35
Stream.grouped(10),
34
36
//Stream.schedule(Schedule.spaced("3 second")),
35
37
Stream.mapEffect(saveChunk, { concurrency: 5 }),
36
-
Stream.runDrain,
38
+
Stream.runDrain
37
39
);
38
40
});
39
41
40
-
type Metadata = typeof MetadataWithFilepathSchema.Type;
41
-
42
42
const saveChunk = Effect.fn("save-chunk")(function* (chunk: Chunk.Chunk<Metadata>) {
43
43
const [files, artists, albums] = yield* Effect.all(
44
44
[createFiles(chunk), createArtists(chunk), createAlbums(chunk)],
45
-
{ concurrency: 3 },
45
+
{ concurrency: 3 }
46
46
);
47
47
48
48
const songs = yield* createSongs(chunk, { files, albums });
···
58
58
albums,
59
59
}),
60
60
],
61
-
{ concurrency: 2 },
61
+
{ concurrency: 2 }
62
62
);
63
63
64
64
return {
···
74
74
lookup: {
75
75
songs: Effect.Effect.Success<ReturnType<typeof createSongs>>;
76
76
artists: Effect.Effect.Success<ReturnType<typeof createArtists>>;
77
-
},
77
+
}
78
78
) {
79
79
const db = yield* DatabaseLive;
80
80
const newSongArtist = Chunk.toArray(chunk)
···
82
82
entry.artists.map((artist) => ({
83
83
artistId: lookup.artists.find((x) => x.name === artist)?.id ?? "",
84
84
songId: lookup.songs.find((x) => x.title === entry.title)?.id ?? "",
85
-
})),
85
+
}))
86
86
)
87
87
.filter((x) => x.songId && x.artistId);
88
88
···
99
99
lookup: {
100
100
albums: Effect.Effect.Success<ReturnType<typeof createAlbums>>;
101
101
artists: Effect.Effect.Success<ReturnType<typeof createArtists>>;
102
-
},
102
+
}
103
103
) {
104
104
const db = yield* DatabaseLive;
105
105
const newAlbumArtists = Chunk.toArray(chunk)
···
121
121
lookup: {
122
122
files: Effect.Effect.Success<ReturnType<typeof createFiles>>;
123
123
albums: Effect.Effect.Success<ReturnType<typeof createAlbums>>;
124
-
},
124
+
}
125
125
) {
126
126
const db = yield* DatabaseLive;
127
127
···
180
180
.values(
181
181
newArtists.map((x) => ({
182
182
name: x,
183
-
})),
183
+
}))
184
184
)
185
185
.onConflictDoNothing()
186
186
.returning();
···
207
207
.values(
208
208
newAlbums.map((x) => ({
209
209
title: x,
210
-
})),
210
+
}))
211
211
)
212
212
.onConflictDoNothing()
213
213
.returning();
···
222
222
223
223
function chunkToUniqueArray<T extends object, U extends string | readonly string[]>(
224
224
chunk: Chunk.Chunk<T>,
225
-
map: (x: T) => U,
225
+
map: (x: T) => U
226
226
) {
227
227
const array = Chunk.toArray(chunk);
228
228
const x = array.flatMap(map);