🧱 Chunk is a download manager for slow and unstable servers
Merge pull request #41 from cuducos/chunk-dir-config
Makes Chunk directory configurable
authored by
Eduardo Cuducos
and committed by
GitHub
b77a5658
76cffe04
+41
-20
+9
cmd/chunk/main.go
+9
cmd/chunk/main.go
···
47
47
chunk.WaitRetry = waitBetweenRetries
48
48
chunk.ChunkSize = chunkSize
49
49
chunk.RestartDownloads = restartDownloads
50
+
chunk.ProgressDir = progressDir
50
51
prog := newProgress()
51
52
for status := range chunk.Download(args...) {
52
53
if status.Error != nil {
···
67
68
chunkSize int64
68
69
waitBetweenRetries time.Duration
69
70
restartDownloads bool
71
+
progressDir string
70
72
)
71
73
72
74
func init() {
···
77
79
rootCmd.Flags().Int64VarP(&chunkSize, "chunk-size", "s", chunk.DefaultChunkSize, "maximum size of each HTTP request done using the content range header.")
78
80
rootCmd.Flags().IntVarP(&concurrencyPerServer, "concurrency-per-server", "c", chunk.DefaultConcurrencyPerServer, "controls the max number of concurrent connections opened to the same server.")
79
81
rootCmd.Flags().BoolVarP(&restartDownloads, "force-restart", "f", chunk.DefaultRestartDownload, "restart previous downloads, ignoring where they were stopped")
82
+
rootCmd.Flags().StringVarP(
83
+
&progressDir,
84
+
"progress-directory",
85
+
"p",
86
+
"",
87
+
fmt.Sprintf("directory where to track progress; %s under user's home directory if blank, or CHUNK_DIR environment variable", chunk.DefaultChunkDir),
88
+
)
80
89
}
81
90
82
91
func main() {
+5
-1
downloader.go
+5
-1
downloader.go
···
95
95
// RestartDownloads controls whether or not to continue the download of
96
96
// previous download attempts, skipping chunks alreadt downloaded.
97
97
RestartDownloads bool
98
+
99
+
// ProgressDir is the directory where Chunk keeps track of each chunk
100
+
// downloaded of each file.
101
+
ProgressDir string
98
102
}
99
103
100
104
type chunk struct{ start, end int64 }
···
248
252
return
249
253
}
250
254
chunks := d.chunks(t)
251
-
p, err := newProgress(s.DownloadedFilePath, s.URL, d.ChunkSize, len(chunks), d.RestartDownloads)
255
+
p, err := newProgress(s.DownloadedFilePath, d.ProgressDir, s.URL, d.ChunkSize, len(chunks), d.RestartDownloads)
252
256
if err != nil {
253
257
s.Error = fmt.Errorf("could not creat a progress file: %w", err)
254
258
ch <- s
+19
-11
progress.go
+19
-11
progress.go
···
11
11
"sync/atomic"
12
12
)
13
13
14
-
const defaultChunkDir = ".chunk"
14
+
// DefaultChunkDir is the directory where Chunk keeps track of each chunk
15
+
// downloaded of each file. It us created under the user's home directory by
16
+
// default. It can be replaced by the environment variable CHUNK_DIR.
17
+
const DefaultChunkDir = ".chunk"
15
18
16
19
// get the chunk directory under user's home directory
17
20
// TODO: make it configurable (maybe an envvar?)
18
-
func getChunkDirectory() (string, error) {
19
-
u, err := user.Current()
20
-
if err != nil {
21
-
return "", fmt.Errorf("could not get current user: %w", err)
21
+
func getChunkProgressDir(dir string) (string, error) {
22
+
if dir == "" {
23
+
dir = os.Getenv("CHUNK_DIR")
22
24
}
23
-
d := filepath.Join(u.HomeDir, defaultChunkDir)
24
-
if err := os.MkdirAll(d, 0755); err != nil {
25
-
return "", fmt.Errorf("could not create chunk's directory %s: %w", d, err)
25
+
if dir == "" {
26
+
u, err := user.Current()
27
+
if err != nil {
28
+
return "", fmt.Errorf("could not get current user: %w", err)
29
+
}
30
+
dir = filepath.Join(u.HomeDir, DefaultChunkDir)
31
+
}
32
+
if err := os.MkdirAll(dir, 0755); err != nil {
33
+
return "", fmt.Errorf("could not create chunk's directory %s: %w", dir, err)
26
34
}
27
-
return d, nil
35
+
return dir, nil
28
36
}
29
37
30
38
type progress struct {
···
147
155
return downloaded
148
156
}
149
157
150
-
func newProgress(path, url string, chunkSize int64, chunks int, restart bool) (*progress, error) {
151
-
dir, err := getChunkDirectory()
158
+
func newProgress(path, dir string, url string, chunkSize int64, chunks int, restart bool) (*progress, error) {
159
+
dir, err := getChunkProgressDir(dir)
152
160
if err != nil {
153
161
return nil, fmt.Errorf("could not get chunk's directory: %w", err)
154
162
}
+8
-8
progress_test.go
+8
-8
progress_test.go
···
10
10
func TestProgress_FromScratch(t *testing.T) {
11
11
tmp := t.TempDir()
12
12
name := filepath.Join(tmp, "chunk.zip")
13
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
13
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
14
14
if err != nil {
15
15
t.Errorf("expected no error creating the progress, got %s", err)
16
16
}
···
43
43
func TestProgress_ParallelComplete(t *testing.T) {
44
44
tmp := t.TempDir()
45
45
name := filepath.Join(tmp, "chunk.zip")
46
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 5, 2048, false)
46
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 2048, false)
47
47
if err != nil {
48
48
t.Errorf("expected no error creating the progress, got %s", err)
49
49
}
···
74
74
func TestProgress_FromFile(t *testing.T) {
75
75
tmp := t.TempDir()
76
76
name := filepath.Join(tmp, "chunk.zip")
77
-
old, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
77
+
old, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
78
78
if err != nil {
79
79
t.Errorf("expected no error creating the old progress, got %s", err)
80
80
}
81
81
old.done(1, 3)
82
82
old.close()
83
83
84
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
84
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
85
85
if err != nil {
86
86
t.Errorf("expected no error creating the progress, got %s", err)
87
87
}
···
111
111
func TestProgress_FromFileWithInvalidChunkSize(t *testing.T) {
112
112
tmp := t.TempDir()
113
113
name := filepath.Join(tmp, "chunk.zip")
114
-
old, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
114
+
old, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
115
115
if err != nil {
116
116
t.Errorf("expected no error creating the old progress, got %s", err)
117
117
}
118
118
old.done(1, 3)
119
119
old.close()
120
120
121
-
if _, err := newProgress(name, "https://test.etc/chunk.zip", 10, 3, false); err == nil {
121
+
if _, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 10, 3, false); err == nil {
122
122
t.Error("expected error creating the progress with different chunk size")
123
123
}
124
124
}
···
126
126
func TestProgress_FromFileWithRestart(t *testing.T) {
127
127
tmp := t.TempDir()
128
128
name := filepath.Join(tmp, "chunk.zip")
129
-
old, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
129
+
old, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
130
130
if err != nil {
131
131
t.Errorf("expected no error creating the old progress, got %s", err)
132
132
}
133
133
old.done(1, 3)
134
134
old.close()
135
135
136
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 10, 3, true)
136
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 10, 3, true)
137
137
if err != nil {
138
138
t.Errorf("expected no error creating the progress, got %s", err)
139
139
}