🧱 Chunk is a download manager for slow and unstable servers
Introducing command-line options to chunk binary
+66
-16
+44
-8
cmd/chunk/main.go
+44
-8
cmd/chunk/main.go
···
4
4
"fmt"
5
5
"log"
6
6
"os"
7
+
"time"
7
8
8
9
"github.com/cuducos/chunk"
10
+
"github.com/spf13/cobra"
9
11
)
10
12
11
-
func main() {
12
-
chunk := chunk.DefaultDownloader()
13
-
prog := newProgress()
14
-
for status := range chunk.Download(os.Args[1:len(os.Args)]...) {
15
-
if status.Error != nil {
16
-
log.Fatal(status.Error)
13
+
var rootCmd = &cobra.Command{
14
+
Use: "chunk",
15
+
Short: "Download tool for slow and unstable servers",
16
+
Long: `The idea of the project emerged as it was difficult for Minha Receita to handle the download of 37 files that adds up to just approx. 5Gb. Most of the download solutions out there (e.g. got) seem to be prepared for downloading large files, not for downloading from slow and unstable servers — which is the case at hand.`,
17
+
Run: func(cmd *cobra.Command, args []string) {
18
+
chunk := chunk.DefaultDownloader()
19
+
chunk.TimeoutPerChunk = timeoutChunk
20
+
chunk.MaxParallelDownloadsPerServer = concurrencyPerServer
21
+
chunk.MaxRetriesPerChunk = maxRetriesChunk
22
+
chunk.WaitBetweenRetries = waitBetweenRetries
23
+
chunk.ChunkSize = chunkSize
24
+
prog := newProgress()
25
+
for status := range chunk.Download(os.Args[1:len(os.Args)]...) {
26
+
if status.Error != nil {
27
+
log.Fatal(status.Error)
28
+
}
29
+
prog.update(status)
17
30
}
18
-
prog.update(status)
31
+
fmt.Printf("\r%s\nDownloaded to: %s", prog.String(), os.TempDir())
32
+
},
33
+
}
34
+
35
+
// Flags
36
+
var (
37
+
timeoutChunk time.Duration
38
+
concurrencyPerServer int
39
+
maxRetriesChunk uint
40
+
chunkSize int64
41
+
waitBetweenRetries time.Duration
42
+
)
43
+
44
+
func init() {
45
+
rootCmd.Flags().DurationVarP(&timeoutChunk, "timeout", "t", chunk.DefaultTimeoutPerChunk, "timeout for the download of each chunk from each URL.")
46
+
rootCmd.Flags().UintVarP(&maxRetriesChunk, "max-retries", "r", chunk.DefaultMaxRetriesPerChunk, "maximum number of retries for each chunk.")
47
+
rootCmd.Flags().DurationVarP(&waitBetweenRetries, "wait-between-retries", "w", chunk.DefaultWaitBetweenRetries, "pause before retrying an HTTP request that has failed.")
48
+
rootCmd.Flags().Int64VarP(&chunkSize, "chunk-size", "s", chunk.DefaultChunkSize, "maximum size of each HTTP request done using the content range header.")
49
+
rootCmd.Flags().IntVarP(&concurrencyPerServer, "concurrency-per-server", "c", chunk.DefaultMaxParallelDownloadsPerServer, "controls the max number of concurrent connections opened to the same server.")
50
+
}
51
+
52
+
func main() {
53
+
if err := rootCmd.Execute(); err != nil {
54
+
fmt.Println(err)
55
+
os.Exit(1)
19
56
}
20
-
fmt.Printf("\r%s\nDownloaded to: %s", prog.String(), os.TempDir())
21
57
}
+5
-6
downloader.go
+5
-6
downloader.go
···
19
19
DefaultMaxParallelDownloadsPerServer = 8
20
20
DefaultMaxRetriesPerChunk = 5
21
21
DefaultChunkSize = 8192
22
-
DefaultWaitBetweenRetries = 0 * time.Minute
22
+
DefaultWaitBetweenRetries = 1 * time.Second
23
23
)
24
24
25
25
// DownloadStatus is the data propagated via the channel sent back to the user
···
64
64
// the the download of every file).
65
65
TimeoutPerChunk time.Duration
66
66
67
-
// MaxParallelDownloadsPerServer controls how many requests are sent in
68
-
// parallel to the same server. If all the URLs are from the same server
69
-
// this is the total of parallel requests. If the user is downloading files
70
-
// from different servers (including different subdomains), this limit is
71
-
// applied to each server idependently.
67
+
// MaxParallelDownloadsPerServer controls the max number of concurrent
68
+
// connections opened to the same server. If all the URLs are from the same
69
+
// server this is the total of concurrent connections. If the user is downloading
70
+
// files from different servers, this limit is applied to each server idependently.
72
71
MaxParallelDownloadsPerServer int
73
72
74
73
// MaxRetriesPerChunk is the maximum amount of retries for each HTTP request
+9
-2
go.mod
+9
-2
go.mod
···
2
2
3
3
go 1.19
4
4
5
-
require github.com/avast/retry-go v3.0.0+incompatible
5
+
require (
6
+
github.com/avast/retry-go v3.0.0+incompatible
7
+
github.com/spf13/cobra v1.6.1
8
+
)
6
9
7
-
require github.com/stretchr/testify v1.8.1 // indirect
10
+
require (
11
+
github.com/inconshreveable/mousetrap v1.0.1 // indirect
12
+
github.com/spf13/pflag v1.0.5 // indirect
13
+
github.com/stretchr/testify v1.8.1 // indirect
14
+
)
+8
go.sum
+8
go.sum
···
1
1
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
2
2
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
3
+
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
3
4
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4
5
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5
6
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7
+
github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
8
+
github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
6
9
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
7
10
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
11
+
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
12
+
github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
13
+
github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
14
+
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
15
+
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
8
16
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
9
17
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
10
18
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=