🧱 Chunk is a download manager for slow and unstable servers
Merge pull request #32 from cuducos/dir
Adds target directory
authored by
Eduardo Cuducos
and committed by
GitHub
3b8576ae
50891bf8
+49
-5
+29
-1
cmd/chunk/main.go
+29
-1
cmd/chunk/main.go
···
10
10
"github.com/spf13/cobra"
11
11
)
12
12
13
+
func checkOutputDir() error {
14
+
var err error
15
+
if outputDir == "" {
16
+
outputDir, err = os.Getwd()
17
+
if err != nil {
18
+
return fmt.Errorf("could not load current directory: %w", err)
19
+
}
20
+
}
21
+
i, err := os.Stat(outputDir)
22
+
if os.IsNotExist(err) {
23
+
return fmt.Errorf("directory %s does not exist", outputDir)
24
+
}
25
+
if err != nil {
26
+
return fmt.Errorf("could not get info for %s: %w", outputDir, err)
27
+
}
28
+
if !i.Mode().IsDir() {
29
+
return fmt.Errorf("%s is not a directory", outputDir)
30
+
}
31
+
return nil
32
+
}
33
+
13
34
var rootCmd = &cobra.Command{
14
35
Use: "chunk",
15
36
Short: "Download tool for slow and unstable servers",
16
37
Long: "Download tool for slow and unstable servers using HTTP range requests, retries per HTTP request (not by file), prevents re-downloading the same content range and supports wait time to give servers time to recover.",
17
-
Run: func(cmd *cobra.Command, args []string) {
38
+
RunE: func(cmd *cobra.Command, args []string) error {
39
+
if err := checkOutputDir(); err != nil {
40
+
return fmt.Errorf("error checking %s directory: %w", outputDir, err)
41
+
}
18
42
chunk := chunk.DefaultDownloader()
43
+
chunk.OutputDir = outputDir
19
44
chunk.Timeout = timeoutChunk
20
45
chunk.ConcurrencyPerServer = concurrencyPerServer
21
46
chunk.MaxRetries = maxRetriesChunk
···
29
54
prog.update(status)
30
55
}
31
56
fmt.Printf("\r%s\nDownloaded to: %s", prog.String(), os.TempDir())
57
+
return nil
32
58
},
33
59
}
34
60
35
61
// Flags
36
62
var (
63
+
outputDir string
37
64
timeoutChunk time.Duration
38
65
concurrencyPerServer int
39
66
maxRetriesChunk uint
···
42
69
)
43
70
44
71
func init() {
72
+
rootCmd.Flags().StringVarP(&outputDir, "output-directory", "d", "", "directory where to save the downloaded files.")
45
73
rootCmd.Flags().DurationVarP(&timeoutChunk, "timeout", "t", chunk.DefaultTimeout, "timeout for the download of each chunk from each URL.")
46
74
rootCmd.Flags().UintVarP(&maxRetriesChunk, "max-retries", "r", chunk.DefaultMaxRetries, "maximum number of retries for each chunk.")
47
75
rootCmd.Flags().DurationVarP(&waitBetweenRetries, "wait-retry", "w", chunk.DefaultWaitRetry, "pause before retrying an HTTP request that has failed.")
+11
-2
downloader.go
+11
-2
downloader.go
···
54
54
// handled, including retries, amoutn of requets, and size of each request, for
55
55
// example.
56
56
type Downloader struct {
57
-
// Client is the HTTP client used for every request needed to download all
57
+
// OutputDir is where the downloaded files will be saved. If not set,
58
+
// defaults to the current working directory.
59
+
OutputDir string
60
+
61
+
// client is the HTTP client used for every request needed to download all
58
62
// the files.
59
63
client *http.Client
60
64
···
222
226
}
223
227
224
228
func (d *Downloader) prepareAndStartDownload(ctx context.Context, url string, ch chan<- DownloadStatus) {
225
-
path := filepath.Join(os.TempDir(), filepath.Base(url))
229
+
path := filepath.Join(d.OutputDir, filepath.Base(url))
226
230
s := DownloadStatus{URL: url, DownloadedFilePath: path}
227
231
t, err := d.getDownloadSize(ctx, url)
228
232
if err != nil {
···
302
306
// NewDownloader creates a downloader with the defalt configuration. Check
303
307
// the constants in this package for their values.
304
308
func DefaultDownloader() *Downloader {
309
+
dir, err := os.Getwd()
310
+
if err != nil {
311
+
dir = ""
312
+
}
305
313
return &Downloader{
314
+
OutputDir: dir,
306
315
Timeout: DefaultTimeout,
307
316
ConcurrencyPerServer: DefaultConcurrencyPerServer,
308
317
MaxRetries: DefaultMaxRetries,
+9
-2
downloader_test.go
+9
-2
downloader_test.go
···
38
38
))
39
39
defer s.Close()
40
40
d := Downloader{
41
+
OutputDir: t.TempDir(),
41
42
Timeout: timeout,
42
43
MaxRetries: 4,
43
44
ConcurrencyPerServer: 1,
···
72
73
))
73
74
defer s.Close()
74
75
75
-
ch := DefaultDownloader().Download(s.URL)
76
+
d := DefaultDownloader()
77
+
d.OutputDir = t.TempDir()
78
+
ch := d.Download(s.URL)
76
79
<-ch // discard the first status (just the file size)
77
80
got := <-ch
78
81
defer os.Remove(got.DownloadedFilePath)
···
138
141
// download
139
142
var got string
140
143
defer os.Remove(got)
141
-
for g := range DefaultDownloader().Download(s.URL + "/archive.zip") {
144
+
d := DefaultDownloader()
145
+
d.OutputDir = t.TempDir()
146
+
for g := range d.Download(s.URL + "/archive.zip") {
142
147
got = g.DownloadedFilePath
143
148
if g.Error != nil {
144
149
t.Errorf("expected no error during the download of the zip archive, got %s", g.Error)
···
191
196
defer s.Close()
192
197
193
198
d := Downloader{
199
+
OutputDir: t.TempDir(),
194
200
Timeout: timeout,
195
201
MaxRetries: 4,
196
202
ConcurrencyPerServer: 1,
···
243
249
))
244
250
defer s.Close()
245
251
d := Downloader{
252
+
OutputDir: t.TempDir(),
246
253
Timeout: timeout,
247
254
MaxRetries: 4,
248
255
ConcurrencyPerServer: 1,