WIP. A little custom music server
fix(sync): prevent silent hangs on large libraries
- Make file parsing per-file fault tolerant: catch errors, skip bad
files, and continue syncing the rest
- Log skipped files with [SKIP] prefix and print summary at sync end
- Reduce saveChunk concurrency from 5 to 1 to avoid SQLite lock contention
- Wrap background sync fiber with error handling so failures are logged
and don't silently stop the repeat loop
- Convert skip list to Set for O(1) lookup
- Fix progress % calculation to use FLAC file count (not total files)
- Reduce console spam: log progress every 50 files instead of per-file
+133
-53
backend/a.out
backend/a.out
This is a binary file and will not be displayed.
+1
-7
backend/src/db/index.ts
+1
-7
backend/src/db/index.ts
···
5
5
import { BunFileSystem, BunPath } from "@effect/platform-bun";
6
6
import * as schema from "./schema";
7
7
8
-
//export const SqlLive = SqliteClient.layer({
9
-
//filename: process.env.DB_URL!,
10
-
//});
11
-
12
-
//const DrizzleLive = SqliteDrizzle.layer.pipe(Layer.provide(SqlLive));
13
-
14
8
export const SqliteLive = SqliteClient.layerConfig({
15
9
filename: Config.string("DB_URL"),
16
10
});
···
28
22
schemaDirectory: "./drizzle",
29
23
}),
30
24
Layer.provide(SqliteLive),
31
-
Layer.provide(BunFileSystem.layer),
25
+
Layer.provide(BunFileSystem.layer)
32
26
);
+56
-13
backend/src/file-parser.ts
+56
-13
backend/src/file-parser.ts
···
1
1
import { Effect, Console, Data, Stream } from "effect";
2
2
import { FileSystem, Path } from "@effect/platform";
3
3
import { FlacService } from "./flac/service";
4
+
import type { MetadataWithFilepathSchema } from "./metadata";
4
5
5
6
const SUPPORTED_EXTENSIONS = ["flac"] as const;
6
7
type SupportedExtension = (typeof SUPPORTED_EXTENSIONS)[number];
···
10
11
message: string;
11
12
}> {}
12
13
14
+
export type SkippedFile = {
15
+
path: string;
16
+
error: string;
17
+
};
18
+
19
+
export type ParseResult =
20
+
| { _tag: "success"; metadata: typeof MetadataWithFilepathSchema.Type }
21
+
| { _tag: "skipped"; file: SkippedFile };
22
+
13
23
export const readDirectory = Effect.fn("read-directory")(function* (dirPath: string, skip: string[] = []) {
14
24
const fs = yield* FileSystem.FileSystem;
15
25
const path = yield* Path.Path;
···
18
28
recursive: true,
19
29
});
20
30
21
-
yield* Console.log(files);
31
+
// Use a Set for O(1) skip lookups
32
+
const skipSet = new Set(skip);
33
+
34
+
// Filter to FLAC files first so progress % is meaningful
35
+
const flacFiles = files
36
+
.map((file) => path.resolve(dirPath, file))
37
+
.filter((file) => SUPPORTED_EXTENSIONS.some((ext) => file.endsWith(ext)))
38
+
.filter((file) => !skipSet.has(file));
22
39
23
-
const total = files.length;
40
+
const total = flacFiles.length;
24
41
let progress = 0;
25
42
26
-
const stream = Stream.fromIterable(files).pipe(
27
-
Stream.map((file) => path.resolve(dirPath, file)),
28
-
Stream.filter((file) => SUPPORTED_EXTENSIONS.some((ext) => file.endsWith(ext)) === true),
29
-
Stream.filter((file) => skip.includes(file) === false),
30
-
Stream.mapEffect((file) => parseFile(file), {
31
-
concurrency: 10,
32
-
}),
33
-
Stream.tap(Console.log),
34
-
Stream.tap(() => {
43
+
yield* Console.log(`Found ${total} FLAC files to process (${files.length} total files in directory)`);
44
+
45
+
const stream = Stream.fromIterable(flacFiles).pipe(
46
+
// Wrap parseFile in Either so errors don't kill the stream
47
+
Stream.mapEffect(
48
+
(file) =>
49
+
parseFile(file).pipe(
50
+
Effect.map(
51
+
(metadata): ParseResult => ({
52
+
_tag: "success",
53
+
metadata,
54
+
})
55
+
),
56
+
Effect.catchAll((err) =>
57
+
Effect.succeed<ParseResult>({
58
+
_tag: "skipped",
59
+
file: {
60
+
path: file,
61
+
error: err instanceof Error ? err.message : String(err),
62
+
},
63
+
})
64
+
)
65
+
),
66
+
{ concurrency: 10 }
67
+
),
68
+
Stream.tap((result) => {
35
69
progress++;
36
-
return Console.log((progress / total) * 100, "%");
70
+
if (result._tag === "skipped") {
71
+
return Console.log(`[SKIP] ${result.file.path}: ${result.file.error}`);
72
+
}
73
+
return Effect.void;
37
74
}),
38
-
//Stream.catchAll((err) => Stream.empty),
75
+
// Log progress every 50 files to reduce console spam
76
+
Stream.tap(() => {
77
+
if (progress % 50 === 0 || progress === total) {
78
+
return Console.log(`Progress: ${progress}/${total} (${((progress / total) * 100).toFixed(1)}%)`);
79
+
}
80
+
return Effect.void;
81
+
})
39
82
);
40
83
41
84
return stream;
+19
-5
backend/src/index.ts
+19
-5
backend/src/index.ts
···
13
13
// asdf
14
14
Layer.provideMerge(DatabaseLive.Default),
15
15
Layer.provideMerge(BunContext.layer),
16
-
Layer.merge(OtelLive),
16
+
Layer.merge(OtelLive)
17
17
);
18
18
19
19
const SHUTDOWN_TIMEOUT_MS = 10_000;
20
20
21
+
const setPragmas = Effect.fn("set-pragmas")(function* () {
22
+
const db = yield* DatabaseLive;
23
+
yield* db.run("PRAGMA journal_mode=WAL;");
24
+
yield* db.run("PRAGMA busy_timeout=5000;");
25
+
});
26
+
21
27
const main = Effect.gen(function* () {
22
28
// Run migrations first
23
29
yield* Effect.log("Running database migrations...");
24
30
const dbUrl = yield* Config.string("DB_URL");
31
+
yield* setPragmas();
32
+
25
33
const db = drizzle(new Database(dbUrl));
26
34
yield* Effect.try(() => migrate(db, { migrationsFolder: "./drizzle" }));
27
35
yield* Effect.log("Migrations completed successfully");
···
42
50
Effect.gen(function* () {
43
51
yield* Effect.log(`Received ${signal}, starting graceful shutdown...`);
44
52
yield* Deferred.succeed(shutdownSignal, undefined);
45
-
}).pipe(Effect.provide(AppLayer)),
53
+
}).pipe(Effect.provide(AppLayer))
46
54
);
47
55
};
48
56
···
52
60
53
61
yield* setupShutdownHandlers;
54
62
55
-
// Start library sync in the background, repeating every 10 minutes
56
-
const syncFiber = yield* Effect.fork(
57
-
syncLibraryStream(folderPath).pipe(Effect.repeat(Schedule.spaced("3 minutes"))),
63
+
// Start library sync in the background, repeating every 3 minutes
64
+
// Wrap with error handling so failures are logged and don't silently stop the repeat loop
65
+
const syncWithErrorHandling = syncLibraryStream(folderPath).pipe(
66
+
Effect.tapErrorCause((cause) =>
67
+
Effect.log(`Sync failed with cause: ${cause}`).pipe(Effect.annotateLogs("level", "error"))
68
+
),
69
+
Effect.catchAllCause((cause) => Effect.log(`Sync run failed, will retry on next schedule. Cause: ${cause}`))
58
70
);
71
+
72
+
const syncFiber = yield* Effect.fork(syncWithErrorHandling.pipe(Effect.repeat(Schedule.spaced("3 minutes"))));
59
73
60
74
// Wait for shutdown signal
61
75
yield* Deferred.await(shutdownSignal);
+50
-24
backend/src/sync-library.ts
+50
-24
backend/src/sync-library.ts
···
1
1
import { Effect, Stream, Chunk, Console } from "effect";
2
2
import { DatabaseLive } from "./db";
3
-
import { readDirectory } from "./file-parser";
3
+
import { readDirectory, type ParseResult, type SkippedFile } from "./file-parser";
4
4
import { inArray } from "drizzle-orm";
5
5
6
6
import { albumTable, artistTable, artistToAlbumTable, fileTable, songTable, songToArtistTable } from "./db/schema";
···
12
12
export const syncLibraryStream = Effect.fn("sync-library-stream")(function* (libraryPath: string) {
13
13
const db = yield* DatabaseLive;
14
14
15
-
//TODO: reaindex old files / updated files
15
+
//TODO: reindex old files / updated files
16
16
const alreadyIndexed = yield* db.query.fileTable.findMany();
17
17
18
18
const stream = yield* readDirectory(
19
19
libraryPath,
20
-
alreadyIndexed.map((x) => x.path),
20
+
alreadyIndexed.map((x) => x.path)
21
21
);
22
22
23
-
let i = 0;
23
+
let successCount = 0;
24
+
const skippedFiles: SkippedFile[] = [];
24
25
25
26
yield* stream.pipe(
27
+
// Separate successes from skipped files
28
+
Stream.tap((result: ParseResult) => {
29
+
if (result._tag === "skipped") {
30
+
skippedFiles.push(result.file);
31
+
return Effect.void;
32
+
}
33
+
return Effect.void;
34
+
}),
35
+
// Filter to only successful parses for DB insertion
36
+
Stream.filter((result): result is Extract<ParseResult, { _tag: "success" }> => result._tag === "success"),
37
+
Stream.map((result) => result.metadata),
26
38
Stream.tap((x) =>
27
39
Console.log(
28
40
"Found",
···
32
44
"by",
33
45
colors.FgYellow,
34
46
x.artists.join(","),
35
-
colors.Reset,
36
-
),
47
+
colors.Reset
48
+
)
37
49
),
38
50
Stream.tap(() => {
39
-
i++;
51
+
successCount++;
40
52
return Effect.void;
41
53
}),
42
54
Stream.grouped(10),
43
-
//Stream.schedule(Schedule.spaced("3 second")),
44
-
Stream.mapEffect(saveChunk, { concurrency: 5 }),
45
-
Stream.runDrain,
55
+
// Reduce concurrency to 1 to avoid SQLite lock contention
56
+
Stream.mapEffect(saveChunk, { concurrency: 1 }),
57
+
Stream.runDrain
46
58
);
47
59
48
-
yield* Console.log(`Total found: ${i}`);
60
+
// Print summary
61
+
yield* Console.log(`\n=== Sync Summary ===`);
62
+
yield* Console.log(`Successfully imported: ${successCount} files`);
63
+
yield* Console.log(`Skipped (errors): ${skippedFiles.length} files`);
64
+
65
+
if (skippedFiles.length > 0) {
66
+
yield* Console.log(`\nFirst ${Math.min(10, skippedFiles.length)} skipped files:`);
67
+
for (const file of skippedFiles.slice(0, 10)) {
68
+
yield* Console.log(` - ${file.path}: ${file.error}`);
69
+
}
70
+
if (skippedFiles.length > 10) {
71
+
yield* Console.log(` ... and ${skippedFiles.length - 10} more`);
72
+
}
73
+
}
74
+
yield* Console.log(`====================\n`);
49
75
});
50
76
51
77
const saveChunk = Effect.fn("save-chunk")(function* (chunk: Chunk.Chunk<Metadata>) {
52
78
const [files, artists, albums] = yield* Effect.all(
53
79
[createFiles(chunk), createArtists(chunk), createAlbums(chunk)],
54
-
{ concurrency: 3 },
80
+
{ concurrency: 3 }
55
81
);
56
82
57
83
const songs = yield* createSongs(chunk, { files, albums });
···
67
93
albums,
68
94
}),
69
95
],
70
-
{ concurrency: 2 },
96
+
{ concurrency: 2 }
71
97
);
72
98
73
99
return {
···
83
109
lookup: {
84
110
songs: Effect.Effect.Success<ReturnType<typeof createSongs>>;
85
111
artists: Effect.Effect.Success<ReturnType<typeof createArtists>>;
86
-
},
112
+
}
87
113
) {
88
114
const db = yield* DatabaseLive;
89
115
const newSongArtist = Chunk.toArray(chunk)
···
91
117
entry.artists.map((artist) => ({
92
118
artistId: lookup.artists.find((x) => x.name === artist)?.id ?? "",
93
119
songId: lookup.songs.find((x) => x.title === entry.title)?.id ?? "",
94
-
})),
120
+
}))
95
121
)
96
122
.filter((x) => x.songId && x.artistId);
97
123
···
108
134
lookup: {
109
135
albums: Effect.Effect.Success<ReturnType<typeof createAlbums>>;
110
136
artists: Effect.Effect.Success<ReturnType<typeof createArtists>>;
111
-
},
137
+
}
112
138
) {
113
139
const db = yield* DatabaseLive;
114
140
const newAlbumArtists = Chunk.toArray(chunk)
···
130
156
lookup: {
131
157
files: Effect.Effect.Success<ReturnType<typeof createFiles>>;
132
158
albums: Effect.Effect.Success<ReturnType<typeof createAlbums>>;
133
-
},
159
+
}
134
160
) {
135
161
const db = yield* DatabaseLive;
136
162
···
156
182
.where(
157
183
inArray(
158
184
songTable.fileId,
159
-
newSongs.map((x) => x.fileId),
160
-
),
185
+
newSongs.map((x) => x.fileId)
186
+
)
161
187
);
162
188
});
163
189
···
183
209
.where(
184
210
inArray(
185
211
fileTable.path,
186
-
newFiles.map((x) => x.path),
187
-
),
212
+
newFiles.map((x) => x.path)
213
+
)
188
214
);
189
215
});
190
216
···
202
228
.values(
203
229
newArtists.map((x) => ({
204
230
name: x,
205
-
})),
231
+
}))
206
232
)
207
233
.onConflictDoNothing()
208
234
.returning();
···
230
256
.values(
231
257
newAlbums.map((x) => ({
232
258
title: x,
233
-
})),
259
+
}))
234
260
)
235
261
.onConflictDoNothing()
236
262
.returning();
···
246
272
247
273
function chunkToUniqueArray<T extends object, U extends string | readonly string[]>(
248
274
chunk: Chunk.Chunk<T>,
249
-
map: (x: T) => U,
275
+
map: (x: T) => U
250
276
) {
251
277
const array = Chunk.toArray(chunk);
252
278
const x = array.flatMap(map);
-1
bun.lock
-1
bun.lock
+7
-3
yaak/yaak.rq_KZgY7fLeqp.yaml
+7
-3
yaak/yaak.rq_KZgY7fLeqp.yaml
···
2
2
model: http_request
3
3
id: rq_KZgY7fLeqp
4
4
createdAt: 2025-12-21T07:49:27.879149
5
-
updatedAt: 2025-12-21T07:59:27.020106
5
+
updatedAt: 2025-12-21T16:07:08.926580555
6
6
workspaceId: wk_ojuTjFeKu7
7
7
folderId: null
8
8
authentication: {}
···
18
18
urlParameters:
19
19
- enabled: true
20
20
name: :id
21
-
value: album_01KCZY1PBC5GPS1W8GKCBYV17Q
21
+
value: album_01KD0T5DCWQ857MK16GWK030D3
22
22
id: U3siCTcZtm
23
23
- enabled: true
24
+
name: :id?include=song,album-artist,song-artist
25
+
value: ''
26
+
id: QH7HgP4I8y
27
+
- enabled: true
24
28
name: ''
25
29
value: ''
26
-
id: SUhtSbPSQf
30
+
id: 2epsPgMWCS