Keep using Photos.app like you always do. Attic quietly backs up your originals and edits to an S3 bucket you control. One-way, append-only.
Add batch subdivision on export timeout
When a ladder batch times out, split it in half and retry each half
recursively (max depth 3). This isolates slow iCloud downloads instead
of failing an entire batch of 50 assets. Also adds a retry hint in the
summary when there are failures.
+285
-47
+11
-1
cli/mod.ts
+11
-1
cli/mod.ts
···
135
135
const ladderPath = options.ladder ??
136
136
Deno.env.get("LADDER_PATH") ??
137
137
"ladder";
138
-
const exporter = createLadderExporter(ladderPath);
138
+
const exporter = createLadderExporter(ladderPath, {
139
+
onSubdivide: (size, parts) => {
140
+
if (!options.quiet) {
141
+
console.log(
142
+
` Export timed out (${size} assets) — retrying as ${parts}x${
143
+
Math.ceil(size / parts)
144
+
}...`,
145
+
);
146
+
}
147
+
},
148
+
});
139
149
140
150
await runBackup(assets, manifest, manifestStore, exporter, s3, {
141
151
batchSize: options.batchSize,
+5
-1
cli/src/commands/backup.ts
+5
-1
cli/src/commands/backup.ts
···
320
320
log(`\n ── Complete ──`);
321
321
log(` Uploaded: ${report.uploaded.toLocaleString()}`);
322
322
log(` Failed: ${report.failed.toLocaleString()}`);
323
-
log(` Total: ${formatBytes(report.totalBytes)}\n`);
323
+
log(` Total: ${formatBytes(report.totalBytes)}`);
324
+
if (report.failed > 0) {
325
+
log(`\n Run \`attic backup\` again to retry failed assets.`);
326
+
}
327
+
log();
324
328
logger.complete(report.uploaded, report.failed, report.totalBytes);
325
329
}
326
330
+143
cli/src/export/exporter-subdivide.test.ts
+143
cli/src/export/exporter-subdivide.test.ts
···
1
+
import { assertEquals, assertRejects } from "@std/assert";
2
+
import type { ExportBatchResult } from "./exporter.ts";
3
+
import { exportWithSubdivision } from "./exporter.ts";
4
+
5
+
/** A spawn function that always succeeds, returning one result per UUID. */
6
+
function succeedingSpawn(
7
+
uuids: string[],
8
+
_signal?: AbortSignal,
9
+
): Promise<ExportBatchResult> {
10
+
return Promise.resolve({
11
+
results: uuids.map((uuid) => ({
12
+
uuid,
13
+
path: `/staging/${uuid}.heic`,
14
+
size: 1024,
15
+
sha256: "abc123",
16
+
})),
17
+
errors: [],
18
+
});
19
+
}
20
+
21
+
/** A spawn function that times out N times, then succeeds. */
22
+
function timeoutThenSucceed(
23
+
timesBeforeSuccess: number,
24
+
): (uuids: string[], signal?: AbortSignal) => Promise<ExportBatchResult> {
25
+
let callCount = 0;
26
+
return (uuids, signal) => {
27
+
callCount++;
28
+
if (callCount <= timesBeforeSuccess) {
29
+
return Promise.reject(
30
+
new Error("Ladder subprocess timed out after 300s"),
31
+
);
32
+
}
33
+
return succeedingSpawn(uuids, signal);
34
+
};
35
+
}
36
+
37
+
/** A spawn function that always times out. */
38
+
function alwaysTimeout(
39
+
_uuids: string[],
40
+
_signal?: AbortSignal,
41
+
): Promise<ExportBatchResult> {
42
+
return Promise.reject(
43
+
new Error("Ladder subprocess timed out after 300s"),
44
+
);
45
+
}
46
+
47
+
/** A spawn function that fails with a non-timeout error. */
48
+
function crashingSpawn(
49
+
_uuids: string[],
50
+
_signal?: AbortSignal,
51
+
): Promise<ExportBatchResult> {
52
+
return Promise.reject(new Error("ladder exited with code 1: segfault"));
53
+
}
54
+
55
+
Deno.test("exportWithSubdivision: passes through on success", async () => {
56
+
const result = await exportWithSubdivision(
57
+
succeedingSpawn,
58
+
["a", "b", "c"],
59
+
);
60
+
assertEquals(result.results.length, 3);
61
+
assertEquals(result.errors.length, 0);
62
+
});
63
+
64
+
Deno.test("exportWithSubdivision: subdivides on timeout", async () => {
65
+
const subdivisions: Array<{ size: number; parts: number }> = [];
66
+
// Times out once (full batch of 10), then succeeds on both halves
67
+
const spawn = timeoutThenSucceed(1);
68
+
69
+
const result = await exportWithSubdivision(
70
+
spawn,
71
+
["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"],
72
+
undefined,
73
+
(size, parts) => subdivisions.push({ size, parts }),
74
+
);
75
+
76
+
assertEquals(result.results.length, 10);
77
+
assertEquals(result.errors.length, 0);
78
+
assertEquals(subdivisions.length, 1);
79
+
assertEquals(subdivisions[0].size, 10);
80
+
assertEquals(subdivisions[0].parts, 2);
81
+
});
82
+
83
+
Deno.test("exportWithSubdivision: stops at max depth and reports failures", async () => {
84
+
const subdivisions: Array<{ size: number; parts: number }> = [];
85
+
86
+
const result = await exportWithSubdivision(
87
+
alwaysTimeout,
88
+
["a", "b", "c", "d"],
89
+
undefined,
90
+
(size, parts) => subdivisions.push({ size, parts }),
91
+
);
92
+
93
+
// All should be reported as failed after exhausting subdivision depth
94
+
assertEquals(result.results.length, 0);
95
+
assertEquals(result.errors.length, 4);
96
+
for (const err of result.errors) {
97
+
assertEquals(err.message, "Export timed out after subdivision retries");
98
+
}
99
+
// depth 0 -> 1 -> 2 -> 3 (max), should have subdivided at each level
100
+
assertEquals(subdivisions.length >= 3, true);
101
+
});
102
+
103
+
Deno.test("exportWithSubdivision: non-timeout errors are NOT retried", async () => {
104
+
await assertRejects(
105
+
() =>
106
+
exportWithSubdivision(
107
+
crashingSpawn,
108
+
["a", "b"],
109
+
),
110
+
Error,
111
+
"segfault",
112
+
);
113
+
});
114
+
115
+
Deno.test("exportWithSubdivision: respects abort signal", async () => {
116
+
const controller = new AbortController();
117
+
// Timeout once to trigger subdivision, then abort before second half
118
+
let callCount = 0;
119
+
const spawn = (
120
+
uuids: string[],
121
+
_signal?: AbortSignal,
122
+
): Promise<ExportBatchResult> => {
123
+
callCount++;
124
+
if (callCount === 1) {
125
+
return Promise.reject(
126
+
new Error("Ladder subprocess timed out after 300s"),
127
+
);
128
+
}
129
+
// After first subdivision succeeds, abort before second chunk
130
+
controller.abort();
131
+
return succeedingSpawn(uuids);
132
+
};
133
+
134
+
const result = await exportWithSubdivision(
135
+
spawn,
136
+
["a", "b", "c", "d"],
137
+
controller.signal,
138
+
);
139
+
140
+
// Should have partial results — first half succeeded, second half skipped
141
+
assertEquals(result.results.length, 2);
142
+
assertEquals(result.errors.length, 0);
143
+
});
+126
-45
cli/src/export/exporter.ts
+126
-45
cli/src/export/exporter.ts
···
113
113
/** Default timeout for the ladder subprocess (5 minutes). */
114
114
const LADDER_TIMEOUT_MS = 5 * 60 * 1000;
115
115
116
-
/** Create an exporter that shells out to the ladder binary. */
117
-
export function createLadderExporter(
116
+
/** Maximum subdivision depth when retrying timed-out batches. */
117
+
const MAX_SUBDIVIDE_DEPTH = 3;
118
+
119
+
/** Options for creating a ladder exporter. */
120
+
export interface LadderExporterOptions {
121
+
stagingDir?: string;
122
+
timeoutMs?: number;
123
+
/** Called when a timed-out batch is subdivided for retry. */
124
+
onSubdivide?: (originalSize: number, parts: number) => void;
125
+
}
126
+
127
+
/** Spawn a single ladder process and return the parsed result. */
128
+
async function spawnLadder(
118
129
ladderPath: string,
119
-
stagingDir: string = DEFAULT_STAGING_DIR,
120
-
timeoutMs: number = LADDER_TIMEOUT_MS,
121
-
): Exporter & { stagingDir: string } {
122
-
return {
130
+
uuids: string[],
131
+
stagingDir: string,
132
+
timeoutMs: number,
133
+
signal?: AbortSignal,
134
+
): Promise<ExportBatchResult> {
135
+
await Deno.mkdir(stagingDir, { recursive: true });
136
+
137
+
// PhotoKit expects local identifiers in "UUID/L0/001" format
138
+
const photoKitIds = uuids.map((uuid) => `${uuid}/L0/001`);
139
+
140
+
const request = JSON.stringify({
141
+
uuids: photoKitIds,
123
142
stagingDir,
143
+
});
124
144
125
-
async exportBatch(
126
-
uuids: string[],
127
-
signal?: AbortSignal,
128
-
): Promise<ExportBatchResult> {
129
-
await Deno.mkdir(stagingDir, { recursive: true });
145
+
const cmd = new Deno.Command(ladderPath, {
146
+
stdin: "piped",
147
+
stdout: "piped",
148
+
stderr: "piped",
149
+
});
130
150
131
-
// PhotoKit expects local identifiers in "UUID/L0/001" format
132
-
const photoKitIds = uuids.map((uuid) => `${uuid}/L0/001`);
151
+
const process = cmd.spawn();
152
+
153
+
const writer = process.stdin.getWriter();
154
+
await writer.write(new TextEncoder().encode(request));
155
+
await writer.close();
156
+
157
+
// Race the subprocess against timeout and abort signal
158
+
const result = await raceSubprocess(process, timeoutMs, signal);
159
+
160
+
if (result.code !== 0) {
161
+
const err = new TextDecoder().decode(result.stderr);
162
+
throw new Error(
163
+
`ladder exited with code ${result.code}: ${err.trim()}`,
164
+
);
165
+
}
166
+
167
+
const output = new TextDecoder().decode(result.stdout);
168
+
const parsed: unknown = JSON.parse(output);
169
+
assertExportBatchResult(parsed);
170
+
171
+
// Map PhotoKit identifiers ("UUID/L0/001") back to bare UUIDs
172
+
for (const r of parsed.results) {
173
+
r.uuid = stripLocalIdSuffix(r.uuid);
174
+
}
175
+
for (const e of parsed.errors) {
176
+
e.uuid = stripLocalIdSuffix(e.uuid);
177
+
}
178
+
179
+
return parsed;
180
+
}
133
181
134
-
const request = JSON.stringify({
135
-
uuids: photoKitIds,
136
-
stagingDir,
137
-
});
182
+
function isTimeoutError(error: unknown): boolean {
183
+
return error instanceof Error && /timed out/i.test(error.message);
184
+
}
138
185
139
-
const cmd = new Deno.Command(ladderPath, {
140
-
stdin: "piped",
141
-
stdout: "piped",
142
-
stderr: "piped",
143
-
});
186
+
/** Try a batch; on timeout, split in half and retry each half recursively.
187
+
* Stops at MAX_SUBDIVIDE_DEPTH and reports remaining UUIDs as failed. */
188
+
export async function exportWithSubdivision(
189
+
spawn: (uuids: string[], signal?: AbortSignal) => Promise<ExportBatchResult>,
190
+
uuids: string[],
191
+
signal?: AbortSignal,
192
+
onSubdivide?: (originalSize: number, parts: number) => void,
193
+
depth: number = 0,
194
+
): Promise<ExportBatchResult> {
195
+
try {
196
+
return await spawn(uuids, signal);
197
+
} catch (error: unknown) {
198
+
if (!isTimeoutError(error)) throw error;
199
+
if (depth >= MAX_SUBDIVIDE_DEPTH) {
200
+
// Give up — report all UUIDs in this chunk as failed
201
+
return {
202
+
results: [],
203
+
errors: uuids.map((uuid) => ({
204
+
uuid,
205
+
message: "Export timed out after subdivision retries",
206
+
})),
207
+
};
208
+
}
144
209
145
-
const process = cmd.spawn();
210
+
const mid = Math.ceil(uuids.length / 2);
211
+
const parts = uuids.length <= mid ? 1 : 2;
212
+
onSubdivide?.(uuids.length, parts);
146
213
147
-
const writer = process.stdin.getWriter();
148
-
await writer.write(new TextEncoder().encode(request));
149
-
await writer.close();
214
+
const left = uuids.slice(0, mid);
215
+
const right = uuids.slice(mid);
216
+
const chunks = right.length > 0 ? [left, right] : [left];
150
217
151
-
// Race the subprocess against timeout and abort signal
152
-
const result = await raceSubprocess(process, timeoutMs, signal);
218
+
const combined: ExportBatchResult = { results: [], errors: [] };
219
+
for (const chunk of chunks) {
220
+
if (signal?.aborted) break;
221
+
const result = await exportWithSubdivision(
222
+
spawn,
223
+
chunk,
224
+
signal,
225
+
onSubdivide,
226
+
depth + 1,
227
+
);
228
+
combined.results.push(...result.results);
229
+
combined.errors.push(...result.errors);
230
+
}
231
+
return combined;
232
+
}
233
+
}
153
234
154
-
if (result.code !== 0) {
155
-
const err = new TextDecoder().decode(result.stderr);
156
-
throw new Error(
157
-
`ladder exited with code ${result.code}: ${err.trim()}`,
158
-
);
159
-
}
235
+
/** Create an exporter that shells out to the ladder binary. */
236
+
export function createLadderExporter(
237
+
ladderPath: string,
238
+
opts: LadderExporterOptions = {},
239
+
): Exporter & { stagingDir: string } {
240
+
const stagingDir = opts.stagingDir ?? DEFAULT_STAGING_DIR;
241
+
const timeoutMs = opts.timeoutMs ?? LADDER_TIMEOUT_MS;
242
+
const onSubdivide = opts.onSubdivide;
160
243
161
-
const output = new TextDecoder().decode(result.stdout);
162
-
const parsed: unknown = JSON.parse(output);
163
-
assertExportBatchResult(parsed);
244
+
const spawn = (uuids: string[], signal?: AbortSignal) =>
245
+
spawnLadder(ladderPath, uuids, stagingDir, timeoutMs, signal);
164
246
165
-
// Map PhotoKit identifiers ("UUID/L0/001") back to bare UUIDs
166
-
for (const r of parsed.results) {
167
-
r.uuid = stripLocalIdSuffix(r.uuid);
168
-
}
169
-
for (const e of parsed.errors) {
170
-
e.uuid = stripLocalIdSuffix(e.uuid);
171
-
}
247
+
return {
248
+
stagingDir,
172
249
173
-
return parsed;
250
+
exportBatch(
251
+
uuids: string[],
252
+
signal?: AbortSignal,
253
+
): Promise<ExportBatchResult> {
254
+
return exportWithSubdivision(spawn, uuids, signal, onSubdivide);
174
255
},
175
256
};
176
257
}