🧱 Chunk is a download manager for slow and unstable servers
Finished functionality
+24
-11
+4
-4
cmd/chunk/main.go
+4
-4
cmd/chunk/main.go
···
46
46
chunk.MaxRetries = maxRetriesChunk
47
47
chunk.WaitRetry = waitBetweenRetries
48
48
chunk.ChunkSize = chunkSize
49
+
chunk.RestartDownloads = restartDownloads
49
50
prog := newProgress()
50
-
for status := range chunk.Download(os.Args[1:len(os.Args)]...) {
51
+
for status := range chunk.Download(args...) {
51
52
if status.Error != nil {
52
53
log.Fatal(status.Error)
53
54
}
54
55
prog.update(status)
55
56
}
56
-
fmt.Printf("\r%s\nDownloaded to: %s", prog.String(), os.TempDir())
57
57
return nil
58
58
},
59
59
}
···
66
66
maxRetriesChunk uint
67
67
chunkSize int64
68
68
waitBetweenRetries time.Duration
69
-
continueDownload bool
69
+
restartDownloads bool
70
70
)
71
71
72
72
func init() {
···
76
76
rootCmd.Flags().DurationVarP(&waitBetweenRetries, "wait-retry", "w", chunk.DefaultWaitRetry, "pause before retrying an HTTP request that has failed.")
77
77
rootCmd.Flags().Int64VarP(&chunkSize, "chunk-size", "s", chunk.DefaultChunkSize, "maximum size of each HTTP request done using the content range header.")
78
78
rootCmd.Flags().IntVarP(&concurrencyPerServer, "concurrency-per-server", "c", chunk.DefaultConcurrencyPerServer, "controls the max number of concurrent connections opened to the same server.")
79
-
rootCmd.Flags().BoolVarP(&continueDownload, "continue-download", "n", chunk.DefaultContinueDownloads, "continues previous downloads from where they were stopped")
79
+
rootCmd.Flags().BoolVarP(&restartDownloads, "force-restart", "f", chunk.DefaultRestartDownload, "restart previous downloads, ignoring where they were stopped")
80
80
}
81
81
82
82
func main() {
+9
-7
downloader.go
+9
-7
downloader.go
···
9
9
"path/filepath"
10
10
"strings"
11
11
"sync"
12
+
"sync/atomic"
12
13
"time"
13
14
14
15
"github.com/avast/retry-go"
···
20
21
DefaultMaxRetries = 5
21
22
DefaultChunkSize = 8192
22
23
DefaultWaitRetry = 1 * time.Second
23
-
DefaultContinueDownloads = false
24
+
DefaultRestartDownload = false
24
25
)
25
26
26
27
// DownloadStatus is the data propagated via the channel sent back to the user
···
89
90
// that has failed.
90
91
WaitRetry time.Duration
91
92
92
-
// ContinueDownloads controls whether or not to continue the download of
93
+
// RestartDownloads controls whether or not to continue the download of
93
94
// previous download attempts, skipping chunks alreadt downloaded.
94
-
ContinueDownloads bool
95
+
RestartDownloads bool
95
96
}
96
97
97
98
type chunk struct{ start, end int64 }
···
245
246
return
246
247
}
247
248
chunks := d.chunks(t)
248
-
p, err := newProgress(s.DownloadedFilePath, s.URL, d.ChunkSize, len(chunks), d.ContinueDownloads)
249
+
p, err := newProgress(s.DownloadedFilePath, s.URL, d.ChunkSize, len(chunks), d.RestartDownloads)
249
250
if err != nil {
250
251
s.Error = fmt.Errorf("could not creat a progress file: %w", err)
251
252
ch <- s
···
262
263
ch <- s
263
264
return
264
265
}
266
+
downloadedBytes := p.downloadedBytes()
265
267
for idx, c := range chunks {
266
268
pending, err := p.shouldDownload(idx)
267
269
if err != nil {
···
281
283
ch <- s
282
284
return
283
285
}
284
-
n, err := f.WriteAt(b, c.start)
285
-
if err != nil {
286
+
if _, err := f.WriteAt(b, c.start); err != nil {
286
287
s.Error = fmt.Errorf("error writing to %s: %w", path, err)
287
288
ch <- s
288
289
return
···
292
293
ch <- s
293
294
return
294
295
}
295
-
s.DownloadedFileBytes += int64(n)
296
+
atomic.AddInt64(&downloadedBytes, c.size())
297
+
s.DownloadedFileBytes = downloadedBytes
296
298
ch <- s
297
299
}(c, idx, s)
298
300
}
+11
progress.go
+11
progress.go
···
138
138
return nil // Either not empty or error, suits both cases
139
139
}
140
140
141
+
// calculates the number of bytes downloaded
142
+
func (p *progress) downloadedBytes() int64 {
143
+
var downloaded int64
144
+
for idx := range p.Chunks {
145
+
if p.Chunks[idx] == 1 {
146
+
downloaded += p.ChunkSize
147
+
}
148
+
}
149
+
return downloaded
150
+
}
151
+
141
152
func newProgress(path, url string, chunkSize int64, chunks int, restart bool) (*progress, error) {
142
153
dir, err := getChunkDirectory()
143
154
if err != nil {