🧱 Chunk is a download manager for slow and unstable servers
Makes Chunk directory configurable
Closes #39
+38
-17
+8
cmd/chunk/main.go
+8
cmd/chunk/main.go
···
67
67
chunkSize int64
68
68
waitBetweenRetries time.Duration
69
69
restartDownloads bool
70
+
progressDir string
70
71
)
71
72
72
73
func init() {
···
77
78
rootCmd.Flags().Int64VarP(&chunkSize, "chunk-size", "s", chunk.DefaultChunkSize, "maximum size of each HTTP request done using the content range header.")
78
79
rootCmd.Flags().IntVarP(&concurrencyPerServer, "concurrency-per-server", "c", chunk.DefaultConcurrencyPerServer, "controls the max number of concurrent connections opened to the same server.")
79
80
rootCmd.Flags().BoolVarP(&restartDownloads, "force-restart", "f", chunk.DefaultRestartDownload, "restart previous downloads, ignoring where they were stopped")
81
+
rootCmd.Flags().StringVarP(
82
+
&progressDir,
83
+
"progress-directory",
84
+
"p",
85
+
"",
86
+
fmt.Sprintf("directory where to track progress; %s under user's home directory if blank, or CHUNK_DIR environment variable", chunk.DefaultChunkDir),
87
+
)
80
88
}
81
89
82
90
func main() {
+5
-1
downloader.go
+5
-1
downloader.go
···
95
95
// RestartDownloads controls whether or not to continue the download of
96
96
// previous download attempts, skipping chunks alreadt downloaded.
97
97
RestartDownloads bool
98
+
99
+
// ProgressDir is the directory where Chunk keeps track of each chunk
100
+
// downloaded of each file.
101
+
ProgressDir string
98
102
}
99
103
100
104
type chunk struct{ start, end int64 }
···
248
252
return
249
253
}
250
254
chunks := d.chunks(t)
251
-
p, err := newProgress(s.DownloadedFilePath, s.URL, d.ChunkSize, len(chunks), d.RestartDownloads)
255
+
p, err := newProgress(s.DownloadedFilePath, d.ProgressDir, s.URL, d.ChunkSize, len(chunks), d.RestartDownloads)
252
256
if err != nil {
253
257
s.Error = fmt.Errorf("could not creat a progress file: %w", err)
254
258
ch <- s
+17
-8
progress.go
+17
-8
progress.go
···
11
11
"sync/atomic"
12
12
)
13
13
14
-
const defaultChunkDir = ".chunk"
14
+
// DefaultChunkDir is the directory where Chunk keeps track of each chunk
15
+
// downloaded of each file. It us created under the user's home directory by
16
+
// default. It can be replaced by the environment variable CHUNK_DIR.
17
+
const DefaultChunkDir = ".chunk"
15
18
16
19
// get the chunk directory under user's home directory
17
20
// TODO: make it configurable (maybe an envvar?)
18
-
func getChunkDirectory() (string, error) {
19
-
u, err := user.Current()
20
-
if err != nil {
21
-
return "", fmt.Errorf("could not get current user: %w", err)
21
+
func getChunkDirectory(custom string) (string, error) {
22
+
d := os.Getenv("CHUNK_DIR")
23
+
if custom != "" {
24
+
d = custom
25
+
}
26
+
if d == "" {
27
+
u, err := user.Current()
28
+
if err != nil {
29
+
return "", fmt.Errorf("could not get current user: %w", err)
30
+
}
31
+
d = filepath.Join(u.HomeDir, DefaultChunkDir)
22
32
}
23
-
d := filepath.Join(u.HomeDir, defaultChunkDir)
24
33
if err := os.MkdirAll(d, 0755); err != nil {
25
34
return "", fmt.Errorf("could not create chunk's directory %s: %w", d, err)
26
35
}
···
147
156
return downloaded
148
157
}
149
158
150
-
func newProgress(path, url string, chunkSize int64, chunks int, restart bool) (*progress, error) {
151
-
dir, err := getChunkDirectory()
159
+
func newProgress(path, dir string, url string, chunkSize int64, chunks int, restart bool) (*progress, error) {
160
+
dir, err := getChunkDirectory(dir)
152
161
if err != nil {
153
162
return nil, fmt.Errorf("could not get chunk's directory: %w", err)
154
163
}
+8
-8
progress_test.go
+8
-8
progress_test.go
···
10
10
func TestProgress_FromScratch(t *testing.T) {
11
11
tmp := t.TempDir()
12
12
name := filepath.Join(tmp, "chunk.zip")
13
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
13
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
14
14
if err != nil {
15
15
t.Errorf("expected no error creating the progress, got %s", err)
16
16
}
···
43
43
func TestProgress_ParallelComplete(t *testing.T) {
44
44
tmp := t.TempDir()
45
45
name := filepath.Join(tmp, "chunk.zip")
46
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 5, 2048, false)
46
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 2048, false)
47
47
if err != nil {
48
48
t.Errorf("expected no error creating the progress, got %s", err)
49
49
}
···
74
74
func TestProgress_FromFile(t *testing.T) {
75
75
tmp := t.TempDir()
76
76
name := filepath.Join(tmp, "chunk.zip")
77
-
old, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
77
+
old, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
78
78
if err != nil {
79
79
t.Errorf("expected no error creating the old progress, got %s", err)
80
80
}
81
81
old.done(1, 3)
82
82
old.close()
83
83
84
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
84
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
85
85
if err != nil {
86
86
t.Errorf("expected no error creating the progress, got %s", err)
87
87
}
···
111
111
func TestProgress_FromFileWithInvalidChunkSize(t *testing.T) {
112
112
tmp := t.TempDir()
113
113
name := filepath.Join(tmp, "chunk.zip")
114
-
old, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
114
+
old, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
115
115
if err != nil {
116
116
t.Errorf("expected no error creating the old progress, got %s", err)
117
117
}
118
118
old.done(1, 3)
119
119
old.close()
120
120
121
-
if _, err := newProgress(name, "https://test.etc/chunk.zip", 10, 3, false); err == nil {
121
+
if _, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 10, 3, false); err == nil {
122
122
t.Error("expected error creating the progress with different chunk size")
123
123
}
124
124
}
···
126
126
func TestProgress_FromFileWithRestart(t *testing.T) {
127
127
tmp := t.TempDir()
128
128
name := filepath.Join(tmp, "chunk.zip")
129
-
old, err := newProgress(name, "https://test.etc/chunk.zip", 5, 3, false)
129
+
old, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 5, 3, false)
130
130
if err != nil {
131
131
t.Errorf("expected no error creating the old progress, got %s", err)
132
132
}
133
133
old.done(1, 3)
134
134
old.close()
135
135
136
-
p, err := newProgress(name, "https://test.etc/chunk.zip", 10, 3, true)
136
+
p, err := newProgress(name, tmp, "https://test.etc/chunk.zip", 10, 3, true)
137
137
if err != nil {
138
138
t.Errorf("expected no error creating the progress, got %s", err)
139
139
}