Replace batch subdivision with skip-and-defer strategy

+1 -30

cli/mod.ts

··· 136 136 Deno.env.get("LADDER_PATH") ?? 137 137 "ladder"; 138 138 139 - // Build UUID lookup so subdivision messages can show filenames 140 - const assetByUuid = new Map<string, (typeof assets)[0]>(); 141 - for (const a of assets) { 142 - assetByUuid.set(a.uuid, a); 143 - } 144 - 145 - const { formatBytes } = await import("./src/format.ts"); 146 - const exporter = createLadderExporter(ladderPath, { 147 - onSubdivide: (uuids, parts) => { 148 - if (options.quiet) return; 149 - const names = uuids.map((uuid) => { 150 - const a = assetByUuid.get(uuid); 151 - if (!a) return uuid.substring(0, 8); 152 - const name = a.originalFilename ?? a.filename ?? uuid.substring( 153 - 0, 154 - 8, 155 - ); 156 - const size = a.originalFileSize 157 - ? formatBytes(a.originalFileSize) 158 - : "?"; 159 - return `${name} (${size})`; 160 - }); 161 - console.log( 162 - ` Export timed out — retrying ${uuids.length} assets as ${parts} sub-batches:`, 163 - ); 164 - for (const name of names) { 165 - console.log(` · ${name}`); 166 - } 167 - }, 168 - }); 139 + const exporter = createLadderExporter(ladderPath); 169 140 170 141 await runBackup(assets, manifest, manifestStore, exporter, s3, { 171 142 batchSize: options.batchSize,

+77

cli/src/commands/backup.test.ts

··· 5 5 import { createS3ManifestStore, isBackedUp } from "../manifest/manifest.ts"; 6 6 import { createMockExporter } from "../export/exporter.mock.ts"; 7 7 import { createMockS3Provider } from "../storage/s3-client.mock.ts"; 8 + import type { ExportBatchResult, Exporter } from "../export/exporter.ts"; 8 9 9 10 function makeAsset( 10 11 uuid: string, ··· 277 278 assertEquals(report.uploaded, 1); 278 279 assertEquals(isBackedUp(manifest, "video-1"), true); 279 280 assertEquals(isBackedUp(manifest, "photo-1"), false); 281 + } finally { 282 + await Deno.remove(tmpDir, { recursive: true }); 283 + } 284 + }); 285 + 286 + Deno.test("backup: defers timed-out assets and retries after remaining batches", async () => { 287 + const tmpDir = await Deno.makeTempDir(); 288 + const stagingDir = `${tmpDir}/staging`; 289 + try { 290 + const assets = [ 291 + makeAsset("fast-1"), 292 + makeAsset("slow-1"), 293 + makeAsset("fast-2"), 294 + ]; 295 + 296 + const mockData = new Map([ 297 + [ 298 + "fast-1", 299 + { filename: "IMG_0001.HEIC", data: new TextEncoder().encode("f1") }, 300 + ], 301 + [ 302 + "slow-1", 303 + { filename: "BIG_VIDEO.MOV", data: new TextEncoder().encode("s1") }, 304 + ], 305 + [ 306 + "fast-2", 307 + { filename: "IMG_0003.HEIC", data: new TextEncoder().encode("f2") }, 308 + ], 309 + ]); 310 + 311 + // Exporter that times out on batch calls containing "slow-1", 312 + // but succeeds when "slow-1" is called individually (deferred retry) 313 + let slowRetryCount = 0; 314 + const innerExporter = createMockExporter(mockData, stagingDir); 315 + const timeoutExporter: Exporter & { stagingDir: string } = { 316 + stagingDir, 317 + exportBatch( 318 + uuids: string[], 319 + signal?: AbortSignal, 320 + ): Promise<ExportBatchResult> { 321 + if (uuids.includes("slow-1") && uuids.length > 1) { 322 + // Batch with slow asset: timeout 323 + throw new Error("Ladder subprocess timed out after 300s"); 324 + } 325 + if (uuids.length === 1 && uuids[0] === "slow-1") { 326 + slowRetryCount++; 327 + if (slowRetryCount === 1) { 328 + // First individual retry: also times out (gets deferred) 329 + throw new Error("Ladder subprocess timed out after 300s"); 330 + } 331 + // Second retry (deferred): succeeds 332 + } 333 + return innerExporter.exportBatch(uuids, signal); 334 + }, 335 + }; 336 + 337 + const { s3, manifestStore } = createTestContext(); 338 + const manifest = await manifestStore.load(); 339 + 340 + const report = await runBackup( 341 + assets, 342 + manifest, 343 + manifestStore, 344 + timeoutExporter, 345 + s3, 346 + { batchSize: 3, quiet: true }, 347 + stagingDir, 348 + ); 349 + 350 + // All 3 should be uploaded: fast-1 and fast-2 via individual retry, 351 + // slow-1 via deferred retry 352 + assertEquals(report.uploaded, 3); 353 + assertEquals(report.failed, 0); 354 + assertEquals(isBackedUp(manifest, "fast-1"), true); 355 + assertEquals(isBackedUp(manifest, "fast-2"), true); 356 + assertEquals(isBackedUp(manifest, "slow-1"), true); 280 357 } finally { 281 358 await Deno.remove(tmpDir, { recursive: true }); 282 359 }

+151 -65

cli/src/commands/backup.ts

··· 8 8 } from "@attic/shared"; 9 9 import type { Manifest, ManifestStore } from "../manifest/manifest.ts"; 10 10 import { isBackedUp, markBackedUp } from "../manifest/manifest.ts"; 11 - import type { Exporter } from "../export/exporter.ts"; 12 - import { removeStagedFile } from "../export/exporter.ts"; 11 + import type { ExportBatchResult, Exporter } from "../export/exporter.ts"; 12 + import { isTimeoutError, removeStagedFile } from "../export/exporter.ts"; 13 13 import type { S3Provider } from "../storage/s3-client.ts"; 14 14 import { formatBytes } from "../format.ts"; 15 15 import { startSpinner } from "../spinner.ts"; ··· 173 173 }; 174 174 Deno.addSignalListener("SIGINT", onInterrupt); 175 175 176 - // Process in batches 177 - for (let i = 0; i < pending.length; i += options.batchSize) { 178 - if (signal.aborted) break; 179 - 180 - const batch = pending.slice(i, i + options.batchSize); 181 - const batchUuids = batch.map((a) => a.uuid); 182 - const batchNum = Math.floor(i / options.batchSize) + 1; 183 - const totalBatches = Math.ceil(pending.length / options.batchSize); 184 - 185 - const batchPhotos = batch.filter((a) => a.kind === AssetKind.PHOTO).length; 186 - const batchVideos = batch.length - batchPhotos; 187 - const batchBytes = batch.reduce( 188 - (sum, a) => sum + (a.originalFileSize ?? 0), 189 - 0, 190 - ); 191 - 192 - if (totalBatches > 1) { 193 - const parts = [ 194 - batchPhotos > 0 ? `${batchPhotos} photos` : "", 195 - batchVideos > 0 ? `${batchVideos} videos` : "", 196 - ].filter(Boolean).join(", "); 197 - log( 198 - ` Batch ${batchNum}/${totalBatches} (${parts}, ~${ 199 - formatBytes(batchBytes) 200 - })`, 201 - ); 202 - } 203 - 204 - // Scale timeout based on estimated batch size 205 - if ("setEstimatedBatchBytes" in exporter) { 206 - (exporter as { setEstimatedBatchBytes(n: number): void }) 207 - .setEstimatedBatchBytes(batchBytes); 208 - } 209 - 210 - // 1. Export via ladder 211 - const spinner = options.quiet 212 - ? { stop() {} } 213 - : startSpinner(`Exporting ${batch.length} assets from Photos library...`); 214 - let batchResult; 215 - try { 216 - batchResult = await exporter.exportBatch(batchUuids, signal); 217 - } catch (error: unknown) { 218 - spinner.stop(); 219 - if (signal.aborted) break; 220 - const msg = error instanceof Error ? error.message : String(error); 221 - console.error(` Export failed: ${msg}`); 222 - for (const uuid of batchUuids) { 223 - report.errors.push({ uuid, message: msg }); 224 - report.failed++; 225 - logger.error(uuid, msg); 226 - } 227 - continue; 228 - } 229 - spinner.stop(); 230 - 176 + // Helper: upload exported assets to S3, update manifest and report 177 + async function uploadExported( 178 + batchResult: ExportBatchResult, 179 + ): Promise<void> { 231 180 // Record export errors 232 181 for (const err of batchResult.errors) { 233 182 console.error(` Export error: ${err.uuid} — ${err.message}`); ··· 236 185 logger.error(err.uuid, err.message); 237 186 } 238 187 239 - // 2. Upload each exported file to S3 240 188 for (const exported of batchResult.results) { 241 189 if (signal.aborted) break; 242 190 const asset = assetByUuid.get(exported.uuid); ··· 249 197 const s3Key = originalKey(asset.uuid, asset.dateCreated, ext); 250 198 251 199 try { 252 - // Read file and upload to S3 (with retry for transient failures) 253 200 let fileData: Uint8Array | null = await Deno.readFile(exported.path); 254 201 await withRetry( 255 202 () => s3.putObject(s3Key, fileData!, contentTypeFor(ext)), 256 203 { signal }, 257 204 ); 258 - fileData = null; // Allow GC before metadata upload 205 + fileData = null; 259 206 260 - // Upload metadata JSON (with retry) 261 207 const meta = buildMetadataJson( 262 208 asset, 263 209 s3Key, ··· 277 223 { signal }, 278 224 ); 279 225 280 - // Update manifest 281 226 markBackedUp( 282 227 manifest, 283 228 asset.uuid, ··· 293 238 const typeLabel = asset.kind === AssetKind.PHOTO ? "photo" : "video"; 294 239 logger.uploaded(asset.uuid, name, typeLabel, exported.size); 295 240 296 - // Per-asset progress 297 241 if (!options.quiet) { 298 242 const done = report.uploaded + report.failed; 299 243 const pct = ((done / pending.length) * 100).toFixed(0); ··· 304 248 ); 305 249 } 306 250 307 - // Save manifest periodically (checked per-asset, not per-batch) 308 251 if (sinceLastSave >= options.saveInterval) { 309 252 await manifestStore.save(manifest); 310 253 sinceLastSave = 0; ··· 317 260 report.failed++; 318 261 logger.error(exported.uuid, msg); 319 262 } finally { 320 - // Always clean up staged file, even on interruption or error 321 263 await removeStagedFile(exported.path, resolvedStagingDir); 322 264 } 323 265 } 266 + } 267 + 268 + // Helper: format asset name for log messages 269 + function assetLabel(uuid: string): string { 270 + const a = assetByUuid.get(uuid); 271 + if (!a) return uuid.substring(0, 8); 272 + const name = a.originalFilename ?? a.filename ?? uuid.substring(0, 8); 273 + const size = a.originalFileSize ? formatBytes(a.originalFileSize) : "?"; 274 + const type = a.kind === AssetKind.PHOTO ? "photo" : "video"; 275 + return `${name} (${type}, ${size})`; 276 + } 277 + 278 + // Assets deferred due to individual timeout — retried after all batches 279 + const deferred: string[] = []; 280 + 281 + // Process in batches 282 + for (let i = 0; i < pending.length; i += options.batchSize) { 283 + if (signal.aborted) break; 284 + 285 + const batch = pending.slice(i, i + options.batchSize); 286 + const batchUuids = batch.map((a) => a.uuid); 287 + const batchNum = Math.floor(i / options.batchSize) + 1; 288 + const totalBatches = Math.ceil(pending.length / options.batchSize); 289 + 290 + const batchPhotos = batch.filter((a) => a.kind === AssetKind.PHOTO).length; 291 + const batchVideos = batch.length - batchPhotos; 292 + const batchBytes = batch.reduce( 293 + (sum, a) => sum + (a.originalFileSize ?? 0), 294 + 0, 295 + ); 296 + 297 + if (totalBatches > 1) { 298 + const parts = [ 299 + batchPhotos > 0 ? `${batchPhotos} photos` : "", 300 + batchVideos > 0 ? `${batchVideos} videos` : "", 301 + ].filter(Boolean).join(", "); 302 + log( 303 + ` Batch ${batchNum}/${totalBatches} (${parts}, ~${ 304 + formatBytes(batchBytes) 305 + })`, 306 + ); 307 + } 308 + 309 + // Scale timeout based on estimated batch size 310 + if ("setEstimatedBatchBytes" in exporter) { 311 + (exporter as { setEstimatedBatchBytes(n: number): void }) 312 + .setEstimatedBatchBytes(batchBytes); 313 + } 314 + 315 + // 1. Export via ladder 316 + const spinner = options.quiet 317 + ? { stop() {} } 318 + : startSpinner(`Exporting ${batch.length} assets from Photos library...`); 319 + let batchResult: ExportBatchResult; 320 + try { 321 + batchResult = await exporter.exportBatch(batchUuids, signal); 322 + } catch (error: unknown) { 323 + spinner.stop(); 324 + if (signal.aborted) break; 325 + 326 + // On timeout: retry each asset individually to find the slow ones 327 + if (isTimeoutError(error)) { 328 + log( 329 + ` Batch timed out — retrying ${batch.length} assets individually...`, 330 + ); 331 + const combined: ExportBatchResult = { results: [], errors: [] }; 332 + 333 + for (const uuid of batchUuids) { 334 + if (signal.aborted) break; 335 + const assetBytes = assetByUuid.get(uuid)?.originalFileSize ?? 0; 336 + if ("setEstimatedBatchBytes" in exporter) { 337 + (exporter as { setEstimatedBatchBytes(n: number): void }) 338 + .setEstimatedBatchBytes(assetBytes); 339 + } 340 + try { 341 + const result = await exporter.exportBatch([uuid], signal); 342 + combined.results.push(...result.results); 343 + combined.errors.push(...result.errors); 344 + } catch (innerError: unknown) { 345 + if (signal.aborted) break; 346 + if (isTimeoutError(innerError)) { 347 + log( 348 + ` Deferring ${ 349 + assetLabel(uuid) 350 + } — timed out, will retry after remaining batches`, 351 + ); 352 + deferred.push(uuid); 353 + } else { 354 + const msg = innerError instanceof Error 355 + ? innerError.message 356 + : String(innerError); 357 + report.errors.push({ uuid, message: msg }); 358 + report.failed++; 359 + logger.error(uuid, msg); 360 + } 361 + } 362 + } 363 + 364 + // Upload whatever succeeded from individual retries 365 + await uploadExported(combined); 366 + } else { 367 + // Non-timeout error: fail the whole batch 368 + const msg = error instanceof Error ? error.message : String(error); 369 + console.error(` Export failed: ${msg}`); 370 + for (const uuid of batchUuids) { 371 + report.errors.push({ uuid, message: msg }); 372 + report.failed++; 373 + logger.error(uuid, msg); 374 + } 375 + } 376 + if (totalBatches > 1 && batchNum < totalBatches) log(); 377 + continue; 378 + } 379 + spinner.stop(); 380 + 381 + // 2. Upload exported assets 382 + await uploadExported(batchResult); 324 383 325 384 // Batch separator when multiple batches 326 385 if (totalBatches > 1 && batchNum < totalBatches) { 327 386 log(); 387 + } 388 + } 389 + 390 + // Retry deferred assets with double timeout 391 + if (deferred.length > 0 && !signal.aborted) { 392 + log(); 393 + log(` Retrying ${deferred.length} deferred assets...`); 394 + for (const uuid of deferred) { 395 + if (signal.aborted) break; 396 + const assetBytes = assetByUuid.get(uuid)?.originalFileSize ?? 0; 397 + if ("setEstimatedBatchBytes" in exporter) { 398 + (exporter as { setEstimatedBatchBytes(n: number): void }) 399 + .setEstimatedBatchBytes(assetBytes * 2); 400 + } 401 + try { 402 + const result = await exporter.exportBatch([uuid], signal); 403 + await uploadExported(result); 404 + } catch (retryError: unknown) { 405 + if (signal.aborted) break; 406 + const msg = retryError instanceof Error 407 + ? retryError.message 408 + : String(retryError); 409 + log(` Failed: ${assetLabel(uuid)} — ${msg}`); 410 + report.errors.push({ uuid, message: msg }); 411 + report.failed++; 412 + logger.error(uuid, msg); 413 + } 328 414 } 329 415 } 330 416

+9 -140

cli/src/export/exporter-subdivide.test.ts

··· 1 - import { assertEquals, assertRejects } from "@std/assert"; 2 - import type { ExportBatchResult } from "./exporter.ts"; 3 - import { exportWithSubdivision, timeoutForBytes } from "./exporter.ts"; 4 - 5 - /** A spawn function that always succeeds, returning one result per UUID. */ 6 - function succeedingSpawn( 7 - uuids: string[], 8 - _signal?: AbortSignal, 9 - ): Promise<ExportBatchResult> { 10 - return Promise.resolve({ 11 - results: uuids.map((uuid) => ({ 12 - uuid, 13 - path: `/staging/${uuid}.heic`, 14 - size: 1024, 15 - sha256: "abc123", 16 - })), 17 - errors: [], 18 - }); 19 - } 20 - 21 - /** A spawn function that times out N times, then succeeds. */ 22 - function timeoutThenSucceed( 23 - timesBeforeSuccess: number, 24 - ): (uuids: string[], signal?: AbortSignal) => Promise<ExportBatchResult> { 25 - let callCount = 0; 26 - return (uuids, signal) => { 27 - callCount++; 28 - if (callCount <= timesBeforeSuccess) { 29 - return Promise.reject( 30 - new Error("Ladder subprocess timed out after 300s"), 31 - ); 32 - } 33 - return succeedingSpawn(uuids, signal); 34 - }; 35 - } 36 - 37 - /** A spawn function that always times out. */ 38 - function alwaysTimeout( 39 - _uuids: string[], 40 - _signal?: AbortSignal, 41 - ): Promise<ExportBatchResult> { 42 - return Promise.reject( 43 - new Error("Ladder subprocess timed out after 300s"), 44 - ); 45 - } 46 - 47 - /** A spawn function that fails with a non-timeout error. */ 48 - function crashingSpawn( 49 - _uuids: string[], 50 - _signal?: AbortSignal, 51 - ): Promise<ExportBatchResult> { 52 - return Promise.reject(new Error("ladder exited with code 1: segfault")); 53 - } 1 + import { assertEquals } from "@std/assert"; 2 + import { isTimeoutError, timeoutForBytes } from "./exporter.ts"; 54 3 55 4 Deno.test("timeoutForBytes: scales with size", () => { 56 5 // Small batch: base timeout (5 min) + 1 min for < 100 MB ··· 61 10 assertEquals(timeoutForBytes(0), 5 * 60_000); 62 11 }); 63 12 64 - Deno.test("exportWithSubdivision: passes through on success", async () => { 65 - const result = await exportWithSubdivision( 66 - succeedingSpawn, 67 - ["a", "b", "c"], 13 + Deno.test("isTimeoutError: detects timeout errors", () => { 14 + assertEquals( 15 + isTimeoutError(new Error("Ladder subprocess timed out after 300s")), 16 + true, 68 17 ); 69 - assertEquals(result.results.length, 3); 70 - assertEquals(result.errors.length, 0); 71 - }); 72 - 73 - Deno.test("exportWithSubdivision: subdivides on timeout", async () => { 74 - const subdivisions: Array<{ uuids: string[]; parts: number }> = []; 75 - // Times out once (full batch of 10), then succeeds on both halves 76 - const spawn = timeoutThenSucceed(1); 77 - 78 - const result = await exportWithSubdivision( 79 - spawn, 80 - ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"], 81 - undefined, 82 - (uuids, parts) => subdivisions.push({ uuids: [...uuids], parts }), 83 - ); 84 - 85 - assertEquals(result.results.length, 10); 86 - assertEquals(result.errors.length, 0); 87 - assertEquals(subdivisions.length, 1); 88 - assertEquals(subdivisions[0].uuids.length, 10); 89 - assertEquals(subdivisions[0].parts, 2); 90 - }); 91 - 92 - Deno.test("exportWithSubdivision: stops at max depth and reports failures", async () => { 93 - const subdivisions: Array<{ uuids: string[]; parts: number }> = []; 94 - 95 - const result = await exportWithSubdivision( 96 - alwaysTimeout, 97 - ["a", "b", "c", "d"], 98 - undefined, 99 - (uuids, parts) => subdivisions.push({ uuids: [...uuids], parts }), 100 - ); 101 - 102 - // All should be reported as failed after exhausting subdivision depth 103 - assertEquals(result.results.length, 0); 104 - assertEquals(result.errors.length, 4); 105 - for (const err of result.errors) { 106 - assertEquals(err.message, "Export timed out after subdivision retries"); 107 - } 108 - // depth 0 -> 1 -> 2 -> 3 (max), should have subdivided at each level 109 - assertEquals(subdivisions.length >= 3, true); 110 - }); 111 - 112 - Deno.test("exportWithSubdivision: non-timeout errors are NOT retried", async () => { 113 - await assertRejects( 114 - () => 115 - exportWithSubdivision( 116 - crashingSpawn, 117 - ["a", "b"], 118 - ), 119 - Error, 120 - "segfault", 121 - ); 122 - }); 123 - 124 - Deno.test("exportWithSubdivision: respects abort signal", async () => { 125 - const controller = new AbortController(); 126 - // Timeout once to trigger subdivision, then abort before second half 127 - let callCount = 0; 128 - const spawn = ( 129 - uuids: string[], 130 - _signal?: AbortSignal, 131 - ): Promise<ExportBatchResult> => { 132 - callCount++; 133 - if (callCount === 1) { 134 - return Promise.reject( 135 - new Error("Ladder subprocess timed out after 300s"), 136 - ); 137 - } 138 - // After first subdivision succeeds, abort before second chunk 139 - controller.abort(); 140 - return succeedingSpawn(uuids); 141 - }; 142 - 143 - const result = await exportWithSubdivision( 144 - spawn, 145 - ["a", "b", "c", "d"], 146 - controller.signal, 147 - ); 148 - 149 - // Should have partial results — first half succeeded, second half skipped 150 - assertEquals(result.results.length, 2); 151 - assertEquals(result.errors.length, 0); 18 + assertEquals(isTimeoutError(new Error("something timed out")), true); 19 + assertEquals(isTimeoutError(new Error("connection reset")), false); 20 + assertEquals(isTimeoutError("not an error"), false); 152 21 });

+9 -60

cli/src/export/exporter.ts

··· 116 116 /** Extra timeout per 100 MB of estimated batch size (~1 min per 100 MB). */ 117 117 const TIMEOUT_PER_100MB_MS = 60 * 1000; 118 118 119 - /** Maximum subdivision depth when retrying timed-out batches. */ 120 - const MAX_SUBDIVIDE_DEPTH = 3; 121 - 122 119 /** Calculate timeout based on estimated batch size in bytes. */ 123 120 export function timeoutForBytes(estimatedBytes: number): number { 124 121 const extra = Math.ceil(estimatedBytes / (100 * 1024 * 1024)) * ··· 131 128 stagingDir?: string; 132 129 /** Base timeout in ms (before size scaling). Defaults to 5 min. */ 133 130 baseTimeoutMs?: number; 134 - /** Called when a timed-out batch is subdivided for retry. */ 135 - onSubdivide?: (uuids: string[], parts: number) => void; 136 131 } 137 132 138 133 /** Spawn a single ladder process and return the parsed result. */ ··· 190 185 return parsed; 191 186 } 192 187 193 - function isTimeoutError(error: unknown): boolean { 188 + /** Check whether an error is a ladder timeout. */ 189 + export function isTimeoutError(error: unknown): boolean { 194 190 return error instanceof Error && /timed out/i.test(error.message); 195 191 } 196 192 197 - /** Try a batch; on timeout, split in half and retry each half recursively. 198 - * Stops at MAX_SUBDIVIDE_DEPTH and reports remaining UUIDs as failed. */ 199 - export async function exportWithSubdivision( 200 - spawn: (uuids: string[], signal?: AbortSignal) => Promise<ExportBatchResult>, 201 - uuids: string[], 202 - signal?: AbortSignal, 203 - onSubdivide?: (uuids: string[], parts: number) => void, 204 - depth: number = 0, 205 - ): Promise<ExportBatchResult> { 206 - try { 207 - return await spawn(uuids, signal); 208 - } catch (error: unknown) { 209 - if (!isTimeoutError(error)) throw error; 210 - if (depth >= MAX_SUBDIVIDE_DEPTH) { 211 - // Give up — report all UUIDs in this chunk as failed 212 - return { 213 - results: [], 214 - errors: uuids.map((uuid) => ({ 215 - uuid, 216 - message: "Export timed out after subdivision retries", 217 - })), 218 - }; 219 - } 220 - 221 - const mid = Math.ceil(uuids.length / 2); 222 - const parts = uuids.length <= mid ? 1 : 2; 223 - onSubdivide?.(uuids, parts); 224 - 225 - const left = uuids.slice(0, mid); 226 - const right = uuids.slice(mid); 227 - const chunks = right.length > 0 ? [left, right] : [left]; 228 - 229 - const combined: ExportBatchResult = { results: [], errors: [] }; 230 - for (const chunk of chunks) { 231 - if (signal?.aborted) break; 232 - const result = await exportWithSubdivision( 233 - spawn, 234 - chunk, 235 - signal, 236 - onSubdivide, 237 - depth + 1, 238 - ); 239 - combined.results.push(...result.results); 240 - combined.errors.push(...result.errors); 241 - } 242 - return combined; 243 - } 244 - } 245 - 246 193 /** Create an exporter that shells out to the ladder binary. */ 247 194 export function createLadderExporter( 248 195 ladderPath: string, ··· 250 197 ): Exporter & { stagingDir: string; setEstimatedBatchBytes(n: number): void } { 251 198 const stagingDir = opts.stagingDir ?? DEFAULT_STAGING_DIR; 252 199 const baseTimeoutMs = opts.baseTimeoutMs ?? LADDER_BASE_TIMEOUT_MS; 253 - const onSubdivide = opts.onSubdivide; 254 200 255 201 // Updated by backup.ts before each batch to scale the timeout 256 202 let currentTimeoutMs = baseTimeoutMs; 257 - 258 - const spawn = (uuids: string[], signal?: AbortSignal) => 259 - spawnLadder(ladderPath, uuids, stagingDir, currentTimeoutMs, signal); 260 203 261 204 return { 262 205 stagingDir, ··· 273 216 uuids: string[], 274 217 signal?: AbortSignal, 275 218 ): Promise<ExportBatchResult> { 276 - return exportWithSubdivision(spawn, uuids, signal, onSubdivide); 219 + return spawnLadder( 220 + ladderPath, 221 + uuids, 222 + stagingDir, 223 + currentTimeoutMs, 224 + signal, 225 + ); 277 226 }, 278 227 }; 279 228 }

Configure Feed

Configure Feed