Keep using Photos.app like you always do. Attic quietly backs up your originals and edits to an S3 bucket you control. One-way, append-only.
Improve export timeout UX: detailed logging, size-scaled timeouts, sorted batches
- Log batch composition (photo/video count, estimated size) before export
- On timeout subdivision, list every asset by filename and size so you
can see exactly what's slow
- Scale timeout with estimated batch size (5 min base + 1 min per 100 MB)
- Sort pending assets: photos first, then videos, by size ascending
+103
-23
+2
-1
CHANGELOG.md
+2
-1
CHANGELOG.md
···
10
10
- **Automatic retry on timeout** — when a ladder batch times out (e.g. due to
11
11
iCloud downloads), the batch is split in half and each half retried
12
12
recursively (max depth 3). Only the truly stuck assets end up as failures.
13
-
- **Retry hint** — summary now shows `Run attic backup again to retry failed
13
+
- **Retry hint** — summary now shows
14
+
`Run attic backup again to retry failed
14
15
assets.` when there are failures.
15
16
16
17
## 0.2.0
+26
-6
cli/mod.ts
+26
-6
cli/mod.ts
···
135
135
const ladderPath = options.ladder ??
136
136
Deno.env.get("LADDER_PATH") ??
137
137
"ladder";
138
+
139
+
// Build UUID lookup so subdivision messages can show filenames
140
+
const assetByUuid = new Map<string, (typeof assets)[0]>();
141
+
for (const a of assets) {
142
+
assetByUuid.set(a.uuid, a);
143
+
}
144
+
145
+
const { formatBytes } = await import("./src/format.ts");
138
146
const exporter = createLadderExporter(ladderPath, {
139
-
onSubdivide: (size, parts) => {
140
-
if (!options.quiet) {
141
-
console.log(
142
-
` Export timed out (${size} assets) — retrying as ${parts}x${
143
-
Math.ceil(size / parts)
144
-
}...`,
147
+
onSubdivide: (uuids, parts) => {
148
+
if (options.quiet) return;
149
+
const names = uuids.map((uuid) => {
150
+
const a = assetByUuid.get(uuid);
151
+
if (!a) return uuid.substring(0, 8);
152
+
const name = a.originalFilename ?? a.filename ?? uuid.substring(
153
+
0,
154
+
8,
145
155
);
156
+
const size = a.originalFileSize
157
+
? formatBytes(a.originalFileSize)
158
+
: "?";
159
+
return `${name} (${size})`;
160
+
});
161
+
console.log(
162
+
` Export timed out — retrying ${uuids.length} assets as ${parts} sub-batches:`,
163
+
);
164
+
for (const name of names) {
165
+
console.log(` · ${name}`);
146
166
}
147
167
},
148
168
});
+29
-1
cli/src/commands/backup.ts
+29
-1
cli/src/commands/backup.ts
···
80
80
pending = pending.filter((a) => a.kind === AssetKind.VIDEO);
81
81
}
82
82
83
+
// Sort: photos first, then videos; within each group by size ascending.
84
+
// This keeps fast-to-export photos together and large videos at the end.
85
+
pending.sort((a, b) => {
86
+
if (a.kind !== b.kind) return a.kind - b.kind; // PHOTO=0 before VIDEO=1
87
+
return (a.originalFileSize ?? 0) - (b.originalFileSize ?? 0);
88
+
});
89
+
83
90
// Apply limit
84
91
if (options.limit > 0) {
85
92
pending = pending.slice(0, options.limit);
···
175
182
const batchNum = Math.floor(i / options.batchSize) + 1;
176
183
const totalBatches = Math.ceil(pending.length / options.batchSize);
177
184
185
+
const batchPhotos = batch.filter((a) => a.kind === AssetKind.PHOTO).length;
186
+
const batchVideos = batch.length - batchPhotos;
187
+
const batchBytes = batch.reduce(
188
+
(sum, a) => sum + (a.originalFileSize ?? 0),
189
+
0,
190
+
);
191
+
178
192
if (totalBatches > 1) {
179
-
log(` Batch ${batchNum}/${totalBatches} (${batch.length} assets)`);
193
+
const parts = [
194
+
batchPhotos > 0 ? `${batchPhotos} photos` : "",
195
+
batchVideos > 0 ? `${batchVideos} videos` : "",
196
+
].filter(Boolean).join(", ");
197
+
log(
198
+
` Batch ${batchNum}/${totalBatches} (${parts}, ~${
199
+
formatBytes(batchBytes)
200
+
})`,
201
+
);
202
+
}
203
+
204
+
// Scale timeout based on estimated batch size
205
+
if ("setEstimatedBatchBytes" in exporter) {
206
+
(exporter as { setEstimatedBatchBytes(n: number): void })
207
+
.setEstimatedBatchBytes(batchBytes);
180
208
}
181
209
182
210
// 1. Export via ladder
+15
-6
cli/src/export/exporter-subdivide.test.ts
+15
-6
cli/src/export/exporter-subdivide.test.ts
···
1
1
import { assertEquals, assertRejects } from "@std/assert";
2
2
import type { ExportBatchResult } from "./exporter.ts";
3
-
import { exportWithSubdivision } from "./exporter.ts";
3
+
import { exportWithSubdivision, timeoutForBytes } from "./exporter.ts";
4
4
5
5
/** A spawn function that always succeeds, returning one result per UUID. */
6
6
function succeedingSpawn(
···
52
52
return Promise.reject(new Error("ladder exited with code 1: segfault"));
53
53
}
54
54
55
+
Deno.test("timeoutForBytes: scales with size", () => {
56
+
// Small batch: base timeout (5 min) + 1 min for < 100 MB
57
+
assertEquals(timeoutForBytes(50 * 1024 * 1024), 5 * 60_000 + 60_000);
58
+
// 500 MB batch: base + 5 min
59
+
assertEquals(timeoutForBytes(500 * 1024 * 1024), 5 * 60_000 + 5 * 60_000);
60
+
// 0 bytes: just base
61
+
assertEquals(timeoutForBytes(0), 5 * 60_000);
62
+
});
63
+
55
64
Deno.test("exportWithSubdivision: passes through on success", async () => {
56
65
const result = await exportWithSubdivision(
57
66
succeedingSpawn,
···
62
71
});
63
72
64
73
Deno.test("exportWithSubdivision: subdivides on timeout", async () => {
65
-
const subdivisions: Array<{ size: number; parts: number }> = [];
74
+
const subdivisions: Array<{ uuids: string[]; parts: number }> = [];
66
75
// Times out once (full batch of 10), then succeeds on both halves
67
76
const spawn = timeoutThenSucceed(1);
68
77
···
70
79
spawn,
71
80
["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"],
72
81
undefined,
73
-
(size, parts) => subdivisions.push({ size, parts }),
82
+
(uuids, parts) => subdivisions.push({ uuids: [...uuids], parts }),
74
83
);
75
84
76
85
assertEquals(result.results.length, 10);
77
86
assertEquals(result.errors.length, 0);
78
87
assertEquals(subdivisions.length, 1);
79
-
assertEquals(subdivisions[0].size, 10);
88
+
assertEquals(subdivisions[0].uuids.length, 10);
80
89
assertEquals(subdivisions[0].parts, 2);
81
90
});
82
91
83
92
Deno.test("exportWithSubdivision: stops at max depth and reports failures", async () => {
84
-
const subdivisions: Array<{ size: number; parts: number }> = [];
93
+
const subdivisions: Array<{ uuids: string[]; parts: number }> = [];
85
94
86
95
const result = await exportWithSubdivision(
87
96
alwaysTimeout,
88
97
["a", "b", "c", "d"],
89
98
undefined,
90
-
(size, parts) => subdivisions.push({ size, parts }),
99
+
(uuids, parts) => subdivisions.push({ uuids: [...uuids], parts }),
91
100
);
92
101
93
102
// All should be reported as failed after exhausting subdivision depth
+31
-9
cli/src/export/exporter.ts
+31
-9
cli/src/export/exporter.ts
···
110
110
return slashIndex === -1 ? id : id.substring(0, slashIndex);
111
111
}
112
112
113
-
/** Default timeout for the ladder subprocess (5 minutes). */
114
-
const LADDER_TIMEOUT_MS = 5 * 60 * 1000;
113
+
/** Base timeout for the ladder subprocess (5 minutes). */
114
+
const LADDER_BASE_TIMEOUT_MS = 5 * 60 * 1000;
115
+
116
+
/** Extra timeout per 100 MB of estimated batch size (~1 min per 100 MB). */
117
+
const TIMEOUT_PER_100MB_MS = 60 * 1000;
115
118
116
119
/** Maximum subdivision depth when retrying timed-out batches. */
117
120
const MAX_SUBDIVIDE_DEPTH = 3;
118
121
122
+
/** Calculate timeout based on estimated batch size in bytes. */
123
+
export function timeoutForBytes(estimatedBytes: number): number {
124
+
const extra = Math.ceil(estimatedBytes / (100 * 1024 * 1024)) *
125
+
TIMEOUT_PER_100MB_MS;
126
+
return LADDER_BASE_TIMEOUT_MS + extra;
127
+
}
128
+
119
129
/** Options for creating a ladder exporter. */
120
130
export interface LadderExporterOptions {
121
131
stagingDir?: string;
122
-
timeoutMs?: number;
132
+
/** Base timeout in ms (before size scaling). Defaults to 5 min. */
133
+
baseTimeoutMs?: number;
123
134
/** Called when a timed-out batch is subdivided for retry. */
124
-
onSubdivide?: (originalSize: number, parts: number) => void;
135
+
onSubdivide?: (uuids: string[], parts: number) => void;
125
136
}
126
137
127
138
/** Spawn a single ladder process and return the parsed result. */
···
189
200
spawn: (uuids: string[], signal?: AbortSignal) => Promise<ExportBatchResult>,
190
201
uuids: string[],
191
202
signal?: AbortSignal,
192
-
onSubdivide?: (originalSize: number, parts: number) => void,
203
+
onSubdivide?: (uuids: string[], parts: number) => void,
193
204
depth: number = 0,
194
205
): Promise<ExportBatchResult> {
195
206
try {
···
209
220
210
221
const mid = Math.ceil(uuids.length / 2);
211
222
const parts = uuids.length <= mid ? 1 : 2;
212
-
onSubdivide?.(uuids.length, parts);
223
+
onSubdivide?.(uuids, parts);
213
224
214
225
const left = uuids.slice(0, mid);
215
226
const right = uuids.slice(mid);
···
236
247
export function createLadderExporter(
237
248
ladderPath: string,
238
249
opts: LadderExporterOptions = {},
239
-
): Exporter & { stagingDir: string } {
250
+
): Exporter & { stagingDir: string; setEstimatedBatchBytes(n: number): void } {
240
251
const stagingDir = opts.stagingDir ?? DEFAULT_STAGING_DIR;
241
-
const timeoutMs = opts.timeoutMs ?? LADDER_TIMEOUT_MS;
252
+
const baseTimeoutMs = opts.baseTimeoutMs ?? LADDER_BASE_TIMEOUT_MS;
242
253
const onSubdivide = opts.onSubdivide;
254
+
255
+
// Updated by backup.ts before each batch to scale the timeout
256
+
let currentTimeoutMs = baseTimeoutMs;
243
257
244
258
const spawn = (uuids: string[], signal?: AbortSignal) =>
245
-
spawnLadder(ladderPath, uuids, stagingDir, timeoutMs, signal);
259
+
spawnLadder(ladderPath, uuids, stagingDir, currentTimeoutMs, signal);
246
260
247
261
return {
248
262
stagingDir,
263
+
264
+
/** Set the estimated byte size so the timeout scales accordingly. */
265
+
setEstimatedBatchBytes(estimatedBytes: number) {
266
+
currentTimeoutMs = Math.max(
267
+
baseTimeoutMs,
268
+
timeoutForBytes(estimatedBytes),
269
+
);
270
+
},
249
271
250
272
exportBatch(
251
273
uuids: string[],