Keep using Photos.app like you always do. Attic quietly backs up your originals and edits to an S3 bucket you control. One-way, append-only.
refactor: split BackupPipeline upload logic into BackupUpload
BackupPipeline.swift was 536 lines (over the 500-line convention).
Move UploadContext + uploadExported into the existing BackupUpload.swift
alongside uploadSingleAsset. Pure refactor, no behavioral change.
Prep work for adaptive-concurrency additions in the next release.
+212
-213
-213
Sources/AtticCore/BackupPipeline.swift
-213
Sources/AtticCore/BackupPipeline.swift
···
321
321
}
322
322
323
323
// swiftlint:enable cyclomatic_complexity function_body_length
324
-
325
-
// MARK: - Upload context and helper
326
-
327
-
/// Bundles the non-mutating dependencies for uploadExported, reducing parameter count
328
-
/// and making the recursive retry call less error-prone.
329
-
private struct UploadContext {
330
-
let assetByUUID: [String: AssetInfo]
331
-
let s3: any S3Providing
332
-
let manifestStore: any ManifestStoring
333
-
let saveInterval: Int
334
-
let concurrency: Int
335
-
let progress: any BackupProgressDelegate
336
-
let networkMonitor: (any NetworkMonitoring)?
337
-
let networkTimeout: Duration
338
-
let maxPauseRetries: Int
339
-
}
340
-
341
-
// swiftlint:disable:next function_body_length cyclomatic_complexity
342
-
private func uploadExported(
343
-
_ batchResult: ExportResponse,
344
-
ctx: UploadContext,
345
-
manifest: inout Manifest,
346
-
report: inout BackupReport,
347
-
sinceLastSave: inout Int,
348
-
pauseRetryCount: Int = 0,
349
-
) async throws {
350
-
// Record export errors
351
-
for err in batchResult.errors {
352
-
let filename = ctx.assetByUUID[err.uuid]?.originalFilename ?? err.uuid
353
-
ctx.progress.assetFailed(uuid: err.uuid, filename: filename, message: err.message)
354
-
report.appendError(uuid: err.uuid, message: err.message)
355
-
report.failed += 1
356
-
}
357
-
358
-
let exports = batchResult.results
359
-
if exports.isEmpty { return }
360
-
361
-
// Build upload inputs (compute S3 keys on the caller's task to propagate errors)
362
-
var inputs: [UploadInput] = []
363
-
for exported in exports {
364
-
guard let asset = ctx.assetByUUID[exported.uuid] else { continue }
365
-
let ext = S3Paths.extensionFromUTIOrFilename(
366
-
uti: asset.uniformTypeIdentifier,
367
-
filename: asset.originalFilename ?? "unknown",
368
-
)
369
-
let s3Key = try S3Paths.originalKey(
370
-
uuid: asset.uuid,
371
-
dateCreated: asset.creationDate,
372
-
extension: ext,
373
-
)
374
-
inputs.append(UploadInput(exported: exported, asset: asset, s3Key: s3Key, ext: ext))
375
-
}
376
-
377
-
// Track inputs by UUID for retry lookup
378
-
let inputByUUID = Dictionary(uniqueKeysWithValues: inputs.map { ($0.exported.uuid, $0) })
379
-
380
-
// Concurrent uploads with bounded TaskGroup
381
-
let effectiveConcurrency = max(1, ctx.concurrency)
382
-
var networkPaused = false
383
-
var retryInputs: [UploadInput] = []
384
-
var retryUUIDs: Set<String> = []
385
-
386
-
try await withThrowingTaskGroup(of: UploadResult.self) { group in
387
-
var cursor = 0
388
-
389
-
// Seed initial tasks
390
-
for _ in 0 ..< min(effectiveConcurrency, inputs.count) {
391
-
let input = inputs[cursor]
392
-
cursor += 1
393
-
ctx.progress.assetStarting(
394
-
uuid: input.asset.uuid,
395
-
filename: input.asset.originalFilename ?? "unknown",
396
-
size: actualFileSize(input),
397
-
)
398
-
group.addTask {
399
-
await uploadSingleAsset(input: input, s3: ctx.s3, progress: ctx.progress)
400
-
}
401
-
}
402
-
403
-
// Process results and enqueue next
404
-
for try await result in group {
405
-
try Task.checkCancellation()
406
-
407
-
if let checksum = result.checksum, result.error == nil {
408
-
manifest.markBackedUp(
409
-
uuid: result.uuid,
410
-
s3Key: result.s3Key,
411
-
checksum: checksum,
412
-
size: result.size,
413
-
)
414
-
sinceLastSave += 1
415
-
report.uploaded += 1
416
-
report.totalBytes += result.size
417
-
ctx.progress.assetUploaded(
418
-
uuid: result.uuid,
419
-
filename: result.filename,
420
-
type: result.type,
421
-
size: result.size,
422
-
)
423
-
424
-
// Periodic manifest save
425
-
if sinceLastSave >= ctx.saveInterval {
426
-
do {
427
-
try await ctx.manifestStore.save(manifest)
428
-
ctx.progress.manifestSaved(entriesCount: manifest.entries.count)
429
-
sinceLastSave = 0
430
-
} catch {
431
-
debugPrint("Periodic manifest save failed: \(error)")
432
-
}
433
-
}
434
-
} else if result.isNetworkDownError,
435
-
let monitor = ctx.networkMonitor,
436
-
await !monitor.isNetworkAvailable
437
-
{ // swiftlint:disable:this opening_brace
438
-
// Network-down failure: queue for retry after recovery
439
-
networkPaused = true
440
-
if let input = inputByUUID[result.uuid] {
441
-
retryInputs.append(input)
442
-
retryUUIDs.insert(result.uuid)
443
-
}
444
-
} else {
445
-
// Permanent or non-network failure
446
-
ctx.progress.assetFailed(
447
-
uuid: result.uuid,
448
-
filename: result.filename,
449
-
message: result.error ?? "Unknown error",
450
-
)
451
-
report.appendError(uuid: result.uuid, message: result.error ?? "Unknown error")
452
-
report.failed += 1
453
-
}
454
-
455
-
// Clean up staged file (skip if queued for retry)
456
-
if !retryUUIDs.contains(result.uuid) {
457
-
try? FileManager.default.removeItem(atPath: result.path)
458
-
}
459
-
460
-
// Enqueue next upload (skip if network is down — let group drain)
461
-
if !networkPaused, cursor < inputs.count {
462
-
let input = inputs[cursor]
463
-
cursor += 1
464
-
ctx.progress.assetStarting(
465
-
uuid: input.asset.uuid,
466
-
filename: input.asset.originalFilename ?? "unknown",
467
-
size: actualFileSize(input),
468
-
)
469
-
group.addTask {
470
-
await uploadSingleAsset(input: input, s3: ctx.s3, progress: ctx.progress)
471
-
}
472
-
}
473
-
}
474
-
475
-
// After drain: queue any remaining un-enqueued inputs for retry
476
-
if networkPaused {
477
-
while cursor < inputs.count {
478
-
retryInputs.append(inputs[cursor])
479
-
cursor += 1
480
-
}
481
-
}
482
-
}
483
-
484
-
// Network pause/resume: wait for recovery and retry
485
-
if networkPaused, !retryInputs.isEmpty {
486
-
guard let monitor = ctx.networkMonitor else { return }
487
-
488
-
// Save manifest before waiting (preserve progress)
489
-
if sinceLastSave > 0 {
490
-
do {
491
-
try await ctx.manifestStore.save(manifest)
492
-
ctx.progress.manifestSaved(entriesCount: manifest.entries.count)
493
-
sinceLastSave = 0
494
-
} catch {
495
-
debugPrint("Pre-pause manifest save failed: \(error)")
496
-
}
497
-
}
498
-
499
-
ctx.progress.backupPaused(reason: "Waiting for network...")
500
-
let recovered = try await monitor.waitForNetwork(timeout: ctx.networkTimeout)
501
-
ctx.progress.backupResumed()
502
-
503
-
if recovered, pauseRetryCount < ctx.maxPauseRetries {
504
-
// Build a synthetic ExportResponse from retry inputs
505
-
let retryResults = retryInputs.map(\.exported)
506
-
let retryResponse = ExportResponse(results: retryResults, errors: [])
507
-
508
-
do {
509
-
try await uploadExported(
510
-
retryResponse, ctx: ctx,
511
-
manifest: &manifest, report: &report,
512
-
sinceLastSave: &sinceLastSave,
513
-
pauseRetryCount: pauseRetryCount + 1,
514
-
)
515
-
} catch {
516
-
// Clean up staged files before propagating
517
-
for input in retryInputs {
518
-
try? FileManager.default.removeItem(atPath: input.exported.path)
519
-
}
520
-
throw error
521
-
}
522
-
} else {
523
-
// Timeout or max retries — record failures
524
-
let reason = recovered
525
-
? "Max network pause retries exceeded"
526
-
: "Network unavailable"
527
-
for input in retryInputs {
528
-
let filename = input.asset.originalFilename ?? input.exported.uuid
529
-
ctx.progress.assetFailed(uuid: input.exported.uuid, filename: filename, message: reason)
530
-
report.appendError(uuid: input.exported.uuid, message: reason)
531
-
report.failed += 1
532
-
try? FileManager.default.removeItem(atPath: input.exported.path)
533
-
}
534
-
}
535
-
}
536
-
}
+212
Sources/AtticCore/BackupUpload.swift
+212
Sources/AtticCore/BackupUpload.swift
···
1
1
import Foundation
2
2
import LadderKit
3
3
4
+
/// Bundles the non-mutating dependencies for uploadExported, reducing parameter count
5
+
/// and making the recursive retry call less error-prone.
6
+
struct UploadContext {
7
+
let assetByUUID: [String: AssetInfo]
8
+
let s3: any S3Providing
9
+
let manifestStore: any ManifestStoring
10
+
let saveInterval: Int
11
+
let concurrency: Int
12
+
let progress: any BackupProgressDelegate
13
+
let networkMonitor: (any NetworkMonitoring)?
14
+
let networkTimeout: Duration
15
+
let maxPauseRetries: Int
16
+
}
17
+
18
+
/// Upload exported assets to S3 with bounded concurrency, and handle network-pause retries.
19
+
// swiftlint:disable:next function_body_length cyclomatic_complexity
20
+
func uploadExported(
21
+
_ batchResult: ExportResponse,
22
+
ctx: UploadContext,
23
+
manifest: inout Manifest,
24
+
report: inout BackupReport,
25
+
sinceLastSave: inout Int,
26
+
pauseRetryCount: Int = 0,
27
+
) async throws {
28
+
// Record export errors
29
+
for err in batchResult.errors {
30
+
let filename = ctx.assetByUUID[err.uuid]?.originalFilename ?? err.uuid
31
+
ctx.progress.assetFailed(uuid: err.uuid, filename: filename, message: err.message)
32
+
report.appendError(uuid: err.uuid, message: err.message)
33
+
report.failed += 1
34
+
}
35
+
36
+
let exports = batchResult.results
37
+
if exports.isEmpty { return }
38
+
39
+
// Build upload inputs (compute S3 keys on the caller's task to propagate errors)
40
+
var inputs: [UploadInput] = []
41
+
for exported in exports {
42
+
guard let asset = ctx.assetByUUID[exported.uuid] else { continue }
43
+
let ext = S3Paths.extensionFromUTIOrFilename(
44
+
uti: asset.uniformTypeIdentifier,
45
+
filename: asset.originalFilename ?? "unknown",
46
+
)
47
+
let s3Key = try S3Paths.originalKey(
48
+
uuid: asset.uuid,
49
+
dateCreated: asset.creationDate,
50
+
extension: ext,
51
+
)
52
+
inputs.append(UploadInput(exported: exported, asset: asset, s3Key: s3Key, ext: ext))
53
+
}
54
+
55
+
// Track inputs by UUID for retry lookup
56
+
let inputByUUID = Dictionary(uniqueKeysWithValues: inputs.map { ($0.exported.uuid, $0) })
57
+
58
+
// Concurrent uploads with bounded TaskGroup
59
+
let effectiveConcurrency = max(1, ctx.concurrency)
60
+
var networkPaused = false
61
+
var retryInputs: [UploadInput] = []
62
+
var retryUUIDs: Set<String> = []
63
+
64
+
try await withThrowingTaskGroup(of: UploadResult.self) { group in
65
+
var cursor = 0
66
+
67
+
// Seed initial tasks
68
+
for _ in 0 ..< min(effectiveConcurrency, inputs.count) {
69
+
let input = inputs[cursor]
70
+
cursor += 1
71
+
ctx.progress.assetStarting(
72
+
uuid: input.asset.uuid,
73
+
filename: input.asset.originalFilename ?? "unknown",
74
+
size: actualFileSize(input),
75
+
)
76
+
group.addTask {
77
+
await uploadSingleAsset(input: input, s3: ctx.s3, progress: ctx.progress)
78
+
}
79
+
}
80
+
81
+
// Process results and enqueue next
82
+
for try await result in group {
83
+
try Task.checkCancellation()
84
+
85
+
if let checksum = result.checksum, result.error == nil {
86
+
manifest.markBackedUp(
87
+
uuid: result.uuid,
88
+
s3Key: result.s3Key,
89
+
checksum: checksum,
90
+
size: result.size,
91
+
)
92
+
sinceLastSave += 1
93
+
report.uploaded += 1
94
+
report.totalBytes += result.size
95
+
ctx.progress.assetUploaded(
96
+
uuid: result.uuid,
97
+
filename: result.filename,
98
+
type: result.type,
99
+
size: result.size,
100
+
)
101
+
102
+
// Periodic manifest save
103
+
if sinceLastSave >= ctx.saveInterval {
104
+
do {
105
+
try await ctx.manifestStore.save(manifest)
106
+
ctx.progress.manifestSaved(entriesCount: manifest.entries.count)
107
+
sinceLastSave = 0
108
+
} catch {
109
+
debugPrint("Periodic manifest save failed: \(error)")
110
+
}
111
+
}
112
+
} else if result.isNetworkDownError,
113
+
let monitor = ctx.networkMonitor,
114
+
await !monitor.isNetworkAvailable
115
+
{ // swiftlint:disable:this opening_brace
116
+
// Network-down failure: queue for retry after recovery
117
+
networkPaused = true
118
+
if let input = inputByUUID[result.uuid] {
119
+
retryInputs.append(input)
120
+
retryUUIDs.insert(result.uuid)
121
+
}
122
+
} else {
123
+
// Permanent or non-network failure
124
+
ctx.progress.assetFailed(
125
+
uuid: result.uuid,
126
+
filename: result.filename,
127
+
message: result.error ?? "Unknown error",
128
+
)
129
+
report.appendError(uuid: result.uuid, message: result.error ?? "Unknown error")
130
+
report.failed += 1
131
+
}
132
+
133
+
// Clean up staged file (skip if queued for retry)
134
+
if !retryUUIDs.contains(result.uuid) {
135
+
try? FileManager.default.removeItem(atPath: result.path)
136
+
}
137
+
138
+
// Enqueue next upload (skip if network is down — let group drain)
139
+
if !networkPaused, cursor < inputs.count {
140
+
let input = inputs[cursor]
141
+
cursor += 1
142
+
ctx.progress.assetStarting(
143
+
uuid: input.asset.uuid,
144
+
filename: input.asset.originalFilename ?? "unknown",
145
+
size: actualFileSize(input),
146
+
)
147
+
group.addTask {
148
+
await uploadSingleAsset(input: input, s3: ctx.s3, progress: ctx.progress)
149
+
}
150
+
}
151
+
}
152
+
153
+
// After drain: queue any remaining un-enqueued inputs for retry
154
+
if networkPaused {
155
+
while cursor < inputs.count {
156
+
retryInputs.append(inputs[cursor])
157
+
cursor += 1
158
+
}
159
+
}
160
+
}
161
+
162
+
// Network pause/resume: wait for recovery and retry
163
+
if networkPaused, !retryInputs.isEmpty {
164
+
guard let monitor = ctx.networkMonitor else { return }
165
+
166
+
// Save manifest before waiting (preserve progress)
167
+
if sinceLastSave > 0 {
168
+
do {
169
+
try await ctx.manifestStore.save(manifest)
170
+
ctx.progress.manifestSaved(entriesCount: manifest.entries.count)
171
+
sinceLastSave = 0
172
+
} catch {
173
+
debugPrint("Pre-pause manifest save failed: \(error)")
174
+
}
175
+
}
176
+
177
+
ctx.progress.backupPaused(reason: "Waiting for network...")
178
+
let recovered = try await monitor.waitForNetwork(timeout: ctx.networkTimeout)
179
+
ctx.progress.backupResumed()
180
+
181
+
if recovered, pauseRetryCount < ctx.maxPauseRetries {
182
+
// Build a synthetic ExportResponse from retry inputs
183
+
let retryResults = retryInputs.map(\.exported)
184
+
let retryResponse = ExportResponse(results: retryResults, errors: [])
185
+
186
+
do {
187
+
try await uploadExported(
188
+
retryResponse, ctx: ctx,
189
+
manifest: &manifest, report: &report,
190
+
sinceLastSave: &sinceLastSave,
191
+
pauseRetryCount: pauseRetryCount + 1,
192
+
)
193
+
} catch {
194
+
// Clean up staged files before propagating
195
+
for input in retryInputs {
196
+
try? FileManager.default.removeItem(atPath: input.exported.path)
197
+
}
198
+
throw error
199
+
}
200
+
} else {
201
+
// Timeout or max retries — record failures
202
+
let reason = recovered
203
+
? "Max network pause retries exceeded"
204
+
: "Network unavailable"
205
+
for input in retryInputs {
206
+
let filename = input.asset.originalFilename ?? input.exported.uuid
207
+
ctx.progress.assetFailed(uuid: input.exported.uuid, filename: filename, message: reason)
208
+
report.appendError(uuid: input.exported.uuid, message: reason)
209
+
report.failed += 1
210
+
try? FileManager.default.removeItem(atPath: input.exported.path)
211
+
}
212
+
}
213
+
}
214
+
}
215
+
4
216
/// Result of uploading a single asset, returned from TaskGroup child tasks.
5
217
struct UploadResult {
6
218
let uuid: String