🧱 Chunk is a download manager for slow and unstable servers
Uses progress tracker
Closes #8
+34
-8
+2
cmd/chunk/main.go
+2
cmd/chunk/main.go
···
66
66
maxRetriesChunk uint
67
67
chunkSize int64
68
68
waitBetweenRetries time.Duration
69
+
continueDownload bool
69
70
)
70
71
71
72
func init() {
···
75
76
rootCmd.Flags().DurationVarP(&waitBetweenRetries, "wait-retry", "w", chunk.DefaultWaitRetry, "pause before retrying an HTTP request that has failed.")
76
77
rootCmd.Flags().Int64VarP(&chunkSize, "chunk-size", "s", chunk.DefaultChunkSize, "maximum size of each HTTP request done using the content range header.")
77
78
rootCmd.Flags().IntVarP(&concurrencyPerServer, "concurrency-per-server", "c", chunk.DefaultConcurrencyPerServer, "controls the max number of concurrent connections opened to the same server.")
79
+
rootCmd.Flags().BoolVarP(&continueDownload, "continue-download", "n", chunk.DefaultContinueDownloads, "continues previous downloads from where they were stopped")
78
80
}
79
81
80
82
func main() {
+32
-8
downloader.go
+32
-8
downloader.go
···
20
20
DefaultMaxRetries = 5
21
21
DefaultChunkSize = 8192
22
22
DefaultWaitRetry = 1 * time.Second
23
+
DefaultContinueDownloads = false
23
24
)
24
25
25
26
// DownloadStatus is the data propagated via the channel sent back to the user
···
87
88
// WaitBetweenRetries is an optional pause before retrying an HTTP request
88
89
// that has failed.
89
90
WaitRetry time.Duration
91
+
92
+
// ContinueDownloads controls whether or not to continue the download of
93
+
// previous download attempts, skipping chunks alreadt downloaded.
94
+
ContinueDownloads bool
90
95
}
91
96
92
-
type chunk struct {
93
-
start int64
94
-
end int64
95
-
}
97
+
type chunk struct{ start, end int64 }
96
98
97
99
func (c chunk) size() int64 { return (c.end + 1) - c.start }
98
100
func (c chunk) rangeHeader() string { return fmt.Sprintf("bytes=%d-%d", c.start, c.end) }
···
242
244
ch <- s
243
245
return
244
246
}
247
+
chunks := d.chunks(t)
248
+
p, err := newProgress(s.DownloadedFilePath, s.URL, d.ChunkSize, len(chunks), d.ContinueDownloads)
249
+
if err != nil {
250
+
s.Error = fmt.Errorf("could not creat a progress file: %w", err)
251
+
ch <- s
252
+
return
253
+
}
245
254
var urlDownload sync.WaitGroup
246
255
defer func() {
247
256
urlDownload.Wait()
257
+
p.close()
248
258
f.Close()
249
259
}()
250
260
if err := f.Truncate(int64(t)); err != nil {
···
252
262
ch <- s
253
263
return
254
264
}
255
-
for _, c := range d.chunks(t) {
265
+
for idx, c := range chunks {
266
+
pending, err := p.shouldDownload(idx)
267
+
if err != nil {
268
+
s.Error = fmt.Errorf("could not determine whether chunk #%d is pending: %w", idx+1, err)
269
+
ch <- s
270
+
return
271
+
}
272
+
if !pending {
273
+
continue
274
+
}
256
275
urlDownload.Add(1)
257
-
go func(c chunk, s DownloadStatus) {
276
+
go func(c chunk, idx int, s DownloadStatus) {
258
277
defer urlDownload.Done()
259
278
b, err := d.downloadChunk(ctx, url, c)
260
279
if err != nil {
261
-
s.Error = err
280
+
s.Error = fmt.Errorf("error downloadinf chunk #%d: %w", idx+1, err)
262
281
ch <- s
263
282
return
264
283
}
···
268
287
ch <- s
269
288
return
270
289
}
290
+
if err := p.done(idx); err != nil {
291
+
s.Error = fmt.Errorf("error checking chunk #%d as done: %w", idx+1, err)
292
+
ch <- s
293
+
return
294
+
}
271
295
s.DownloadedFileBytes += int64(n)
272
296
ch <- s
273
-
}(c, s)
297
+
}(c, idx, s)
274
298
}
275
299
}
276
300