🧱 Chunk is a download manager for slow and unstable servers
Makes timeouts not count towards retry max
+308
-209
+124
-62
downloader.go
+124
-62
downloader.go
···
3
3
import (
4
4
"bytes"
5
5
"context"
6
+
"errors"
6
7
"fmt"
7
8
"log/slog"
9
+
"net"
8
10
"net/http"
9
11
"net/url"
10
12
"os"
···
13
15
"sync"
14
16
"sync/atomic"
15
17
"time"
16
-
17
-
"github.com/avast/retry-go/v4"
18
18
)
19
19
20
20
const (
···
26
26
DefaultRestartDownload = false
27
27
DefaultUserAgent = ""
28
28
)
29
+
30
+
var errChunkTimeout = errors.New("chunk timeout")
31
+
32
+
func isTimeout(err error) bool {
33
+
if errors.Is(err, errChunkTimeout) || errors.Is(err, context.DeadlineExceeded) {
34
+
return true
35
+
}
36
+
if err, ok := err.(net.Error); ok && err.Timeout() {
37
+
return true
38
+
}
39
+
return false
40
+
}
29
41
30
42
// DownloadStatus is the data propagated via the channel sent back to the user
31
43
// and it contains information about the download from each URL.
···
115
127
func (c chunk) rangeHeader() string { return fmt.Sprintf("bytes=%d-%d", c.start, c.end) }
116
128
117
129
func (d *Downloader) downloadChunkWithContext(ctx context.Context, u string, c chunk) ([]byte, error) {
130
+
if ctx.Err() != nil { // if context is already done before making request
131
+
return nil, ctx.Err()
132
+
}
118
133
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
119
134
if err != nil {
120
135
return nil, fmt.Errorf("error creating the request for %s: %w", u, err)
···
143
158
return b.Bytes(), nil
144
159
}
145
160
146
-
func (d *Downloader) downloadChunkWithTimeout(userCtx context.Context, u string, c chunk) ([]byte, error) {
147
-
ctx, cancel := context.WithTimeout(userCtx, d.Timeout) // need to propagate context, which might contain app-specific data.
161
+
func (d *Downloader) downloadChunkWithTimeout(usrCtx context.Context, u string, c chunk) ([]byte, error) {
162
+
ctx, cancel := context.WithTimeout(usrCtx, d.Timeout) // need to propagate context, which might contain app-specific data.
148
163
defer cancel()
149
-
ch := make(chan []byte)
150
-
errs := make(chan error)
164
+
165
+
type result struct {
166
+
data []byte
167
+
err error
168
+
}
169
+
ch := make(chan result, 1)
151
170
go func() {
152
171
b, err := d.downloadChunkWithContext(ctx, u, c)
153
-
if err != nil {
154
-
errs <- err
155
-
return
156
-
}
157
-
ch <- b
172
+
ch <- result{b, err}
173
+
close(ch)
158
174
}()
175
+
159
176
select {
160
-
case <-userCtx.Done():
161
-
cancel()
162
-
return nil, userCtx.Err()
177
+
case <-usrCtx.Done():
178
+
return nil, usrCtx.Err()
163
179
case <-ctx.Done():
164
-
return nil, fmt.Errorf("request to %s ended due to timeout: %w", u, ctx.Err())
165
-
case err := <-errs:
166
-
return nil, fmt.Errorf("request to %s failed: %w", u, err)
167
-
case b := <-ch:
168
-
return b, nil
180
+
if usrCtx.Err() != nil {
181
+
return nil, usrCtx.Err()
182
+
}
183
+
return nil, fmt.Errorf("request to %s ended due to timeout: %w", u, errChunkTimeout)
184
+
case r := <-ch:
185
+
if r.err != nil {
186
+
if usrCtx.Err() != nil {
187
+
return nil, usrCtx.Err()
188
+
}
189
+
if ctx.Err() == context.DeadlineExceeded {
190
+
return nil, fmt.Errorf("request to %s ended due to timeout: %w", u, errChunkTimeout)
191
+
}
192
+
return nil, fmt.Errorf("request to %s failed: %w", u, r.err)
193
+
}
194
+
return r.data, nil
169
195
}
170
196
}
171
197
172
198
func (d *Downloader) getDownloadSize(ctx context.Context, u string) (int64, error) {
173
199
ch := make(chan *http.Response, 1)
174
200
defer close(ch)
175
-
err := retry.Do(
176
-
func() error {
177
-
req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)
178
-
if err != nil {
179
-
return fmt.Errorf("creating the request for %s: %w", u, err)
201
+
var n uint
202
+
for {
203
+
if err := ctx.Err(); err != nil {
204
+
return 0, err
205
+
}
206
+
req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)
207
+
if err != nil {
208
+
return 0, fmt.Errorf("creating the request for %s: %w", u, err)
209
+
}
210
+
if d.UserAgent != "" {
211
+
req.Header.Add("User-Agent", d.UserAgent)
212
+
}
213
+
resp, err := d.Client.Do(req)
214
+
if err != nil {
215
+
if ctx.Err() != nil {
216
+
return 0, ctx.Err()
180
217
}
181
-
if d.UserAgent != "" {
182
-
req.Header.Add("User-Agent", d.UserAgent)
218
+
if isTimeout(err) {
219
+
select {
220
+
case <-ctx.Done():
221
+
return 0, ctx.Err()
222
+
case <-time.After(d.WaitRetry):
223
+
continue
224
+
}
183
225
}
184
-
resp, err := d.Client.Do(req)
185
-
if err != nil {
186
-
return fmt.Errorf("dispatching the request for %s: %w", u, err)
226
+
n++
227
+
if n > d.MaxRetries {
228
+
return 0, fmt.Errorf("error sending get http request to %s: %w", u, err)
187
229
}
188
-
if resp.StatusCode != 200 {
189
-
return fmt.Errorf("got unexpected http response status for %s: %s", u, resp.Status)
230
+
select {
231
+
case <-ctx.Done():
232
+
return 0, ctx.Err()
233
+
case <-time.After(d.WaitRetry):
234
+
continue
190
235
}
191
-
ch <- resp
192
-
return nil
193
-
},
194
-
retry.Attempts(d.MaxRetries),
195
-
retry.MaxDelay(d.WaitRetry),
196
-
)
197
-
if err != nil {
198
-
return 0, fmt.Errorf("error sending get http request to %s: %w", u, err)
199
-
}
200
-
resp := <-ch
201
-
defer func() {
236
+
}
202
237
if err := resp.Body.Close(); err != nil {
203
238
slog.Warn("error closing HTTP response body", "url", u, "error", err)
204
239
}
205
-
}()
240
+
if resp.StatusCode != 200 {
241
+
n++
242
+
if n >= d.MaxRetries {
243
+
return 0, fmt.Errorf("got unexpected http response status for %s: %s", u, resp.Status)
244
+
}
245
+
select {
246
+
case <-ctx.Done():
247
+
return 0, ctx.Err()
248
+
case <-time.After(d.WaitRetry):
249
+
continue
250
+
}
251
+
}
252
+
ch <- resp
253
+
break
254
+
}
255
+
resp := <-ch
206
256
if resp.ContentLength <= 0 {
207
-
var s int64
208
257
r := strings.TrimSpace(resp.Header.Get("Content-Range"))
209
258
if r == "" {
210
259
return 0, fmt.Errorf("could not get content length for %s", u)
211
260
}
212
261
p := strings.Split(r, "/")
262
+
var s int64
213
263
if _, err := fmt.Sscan(p[len(p)-1], &s); err != nil {
214
264
return 0, fmt.Errorf("error parsing content range for %s: %w", u, err)
215
265
}
···
219
269
}
220
270
221
271
func (d *Downloader) downloadChunk(ctx context.Context, u string, c chunk) ([]byte, error) {
222
-
ch := make(chan []byte, 1)
223
-
defer close(ch)
224
-
err := retry.Do(
225
-
func() error {
226
-
b, err := d.downloadChunkWithTimeout(ctx, u, c)
227
-
if err != nil {
228
-
return err
272
+
var n uint
273
+
for {
274
+
if err := ctx.Err(); err != nil {
275
+
return nil, err
276
+
}
277
+
b, err := d.downloadChunkWithTimeout(ctx, u, c)
278
+
if err != nil {
279
+
if ctx.Err() != nil {
280
+
return nil, ctx.Err()
229
281
}
230
-
ch <- b
231
-
return nil
232
-
},
233
-
retry.Attempts(d.MaxRetries),
234
-
retry.MaxDelay(d.WaitRetry),
235
-
)
236
-
if err != nil {
237
-
return nil, fmt.Errorf("error downloading %s: %w", u, err)
282
+
if isTimeout(err) {
283
+
select {
284
+
case <-ctx.Done():
285
+
return nil, ctx.Err()
286
+
case <-time.After(d.WaitRetry):
287
+
continue
288
+
}
289
+
}
290
+
n++
291
+
if n >= d.MaxRetries {
292
+
return nil, fmt.Errorf("error downloading %s: %w", u, err)
293
+
}
294
+
select {
295
+
case <-ctx.Done():
296
+
return nil, ctx.Err()
297
+
case <-time.After(d.WaitRetry):
298
+
continue
299
+
}
300
+
}
301
+
return b, nil
238
302
}
239
-
b := <-ch
240
-
return b, nil
241
303
}
242
304
243
305
func (d *Downloader) chunks(t int64) []chunk {
···
307
369
return
308
370
}
309
371
if !pending {
310
-
s.DownloadedFileBytes = atomic.AddInt64(&downloadedBytes, c.size())
372
+
s.DownloadedFileBytes = atomic.LoadInt64(&downloadedBytes)
311
373
ch <- s
312
374
continue
313
375
}
+183
-134
downloader_test.go
+183
-134
downloader_test.go
···
16
16
"time"
17
17
)
18
18
19
-
func TestDownload_Error(t *testing.T) {
20
-
timeout := 250 * time.Millisecond
21
-
for _, tc := range []struct {
22
-
desc string
23
-
proc func(w http.ResponseWriter)
24
-
}{
25
-
{"failure", func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadRequest) }},
26
-
{"timeout", func(w http.ResponseWriter) { time.Sleep(10 * timeout) }},
27
-
} {
28
-
t.Run(tc.desc, func(t *testing.T) {
29
-
s := httptest.NewServer(http.HandlerFunc(
30
-
func(w http.ResponseWriter, r *http.Request) {
31
-
if r.Method == http.MethodHead {
32
-
w.Header().Add("Content-Length", "2")
33
-
return
34
-
}
35
-
tc.proc(w)
36
-
},
37
-
))
38
-
defer s.Close()
39
-
d := Downloader{
40
-
OutputDir: t.TempDir(),
41
-
Timeout: timeout,
42
-
MaxRetries: 4,
43
-
ConcurrencyPerServer: 1,
44
-
ChunkSize: 1024,
45
-
WaitRetry: 0 * time.Second,
19
+
var timeout = 250 * time.Millisecond
20
+
21
+
func TestDownload_HTTPFailure(t *testing.T) {
22
+
t.Parallel()
23
+
s := httptest.NewServer(http.HandlerFunc(
24
+
func(w http.ResponseWriter, r *http.Request) {
25
+
if r.Method == http.MethodHead {
26
+
w.Header().Add("Content-Length", "2")
27
+
return
46
28
}
47
-
ch := d.Download(s.URL)
48
-
<-ch // discard the first got (just the file size)
49
-
got := <-ch
50
-
if got.Error == nil {
51
-
t.Error("expected an error, but got nil")
29
+
w.WriteHeader(http.StatusBadRequest)
30
+
},
31
+
))
32
+
defer s.Close()
33
+
d := Downloader{
34
+
OutputDir: t.TempDir(),
35
+
Timeout: timeout,
36
+
MaxRetries: 4,
37
+
ConcurrencyPerServer: 1,
38
+
ChunkSize: 1024,
39
+
WaitRetry: 1 * time.Millisecond,
40
+
}
41
+
ch := d.Download(s.URL)
42
+
<-ch // discard the first got (just the file size)
43
+
got := <-ch
44
+
if got.Error == nil {
45
+
t.Error("expected an error, but got nil")
46
+
}
47
+
if _, ok := <-ch; ok {
48
+
t.Error("expected channel closed, but did not get it")
49
+
}
50
+
}
51
+
52
+
func TestDownload_ServerTimeout(t *testing.T) {
53
+
t.Parallel()
54
+
s := httptest.NewServer(http.HandlerFunc(
55
+
func(w http.ResponseWriter, r *http.Request) {
56
+
if r.Method == http.MethodHead {
57
+
w.Header().Add("Content-Length", "2")
58
+
return
52
59
}
53
-
if !strings.Contains(got.Error.Error(), "#4") {
54
-
t.Error("expected #4 (configured number of retries), but did not get it")
55
-
}
56
-
if _, ok := <-ch; ok {
57
-
t.Error("expected channel closed, but did not get it")
58
-
}
59
-
})
60
+
time.Sleep(10 * timeout) // server sleeps, causing client timeout
61
+
},
62
+
))
63
+
defer s.Close()
64
+
d := Downloader{
65
+
OutputDir: t.TempDir(),
66
+
Timeout: timeout,
67
+
MaxRetries: 4,
68
+
ConcurrencyPerServer: 1,
69
+
ChunkSize: 1024,
70
+
WaitRetry: 1 * time.Millisecond,
71
+
}
72
+
// context with timeout to eventually terminate the download.
73
+
ctx, cancel := context.WithTimeout(context.Background(), 3*timeout)
74
+
defer cancel()
75
+
ch := d.DownloadWithContext(ctx, s.URL)
76
+
<-ch // discard the first status (file size)
77
+
got := <-ch
78
+
if got.Error == nil {
79
+
t.Error("expected an error due to context timeout, but got nil")
80
+
}
81
+
if _, ok := <-ch; ok {
82
+
t.Error("expected channel closed, but did not get it")
60
83
}
61
84
}
62
85
63
-
func TestDownload_OkWithDefaultDownloader(t *testing.T) {
86
+
func TestDownload_DefaultDownloader(t *testing.T) {
87
+
t.Parallel()
64
88
s := httptest.NewServer(http.HandlerFunc(
65
89
func(w http.ResponseWriter, r *http.Request) {
66
90
if r.Method == http.MethodHead {
···
113
137
}
114
138
115
139
func TestDownload_ZIPArchive(t *testing.T) {
140
+
t.Parallel()
116
141
tmp := t.TempDir()
117
142
pth := filepath.Join(tmp, "archive.zip")
118
-
expected := make([]byte, 1_000_000)
119
-
for i := range 1_000_000 {
143
+
expected := make([]byte, 2<<20)
144
+
for i := range 2 << 20 {
120
145
expected[i] = byte(97 + rand.Intn(122-97))
121
146
}
122
147
···
201
226
}
202
227
203
228
func TestDownload_Retry(t *testing.T) {
204
-
timeout := 250 * time.Millisecond
205
-
for _, tc := range []struct {
206
-
desc string
207
-
proc func(w http.ResponseWriter)
208
-
}{
209
-
{"failure", func(w http.ResponseWriter) { w.WriteHeader(http.StatusBadRequest) }},
210
-
{"timeout", func(w http.ResponseWriter) { time.Sleep(10 * timeout) }},
211
-
} {
212
-
t.Run(tc.desc, func(t *testing.T) {
213
-
attempts := int32(0)
214
-
s := httptest.NewServer(http.HandlerFunc(
215
-
func(w http.ResponseWriter, r *http.Request) {
216
-
if r.Method == http.MethodHead {
217
-
w.Header().Add("Content-Length", "2")
218
-
return
219
-
}
220
-
if atomic.CompareAndSwapInt32(&attempts, 0, 1) {
221
-
tc.proc(w)
222
-
}
223
-
if _, err := io.WriteString(w, "42"); err != nil {
224
-
t.Errorf("failed to write response: %v", err)
225
-
}
226
-
},
227
-
))
228
-
defer s.Close()
229
-
230
-
d := Downloader{
231
-
OutputDir: t.TempDir(),
232
-
Timeout: timeout,
233
-
MaxRetries: 4,
234
-
ConcurrencyPerServer: 1,
235
-
ChunkSize: 1024,
236
-
WaitRetry: 0 * time.Second,
229
+
t.Parallel()
230
+
var done atomic.Bool
231
+
s := httptest.NewServer(http.HandlerFunc(
232
+
func(w http.ResponseWriter, r *http.Request) {
233
+
if r.Method == http.MethodHead {
234
+
w.Header().Add("Content-Length", "2")
235
+
return
237
236
}
238
-
ch := d.Download(s.URL)
239
-
<-ch // discard the first status (just the file size)
240
-
got := <-ch
241
-
if got.Error != nil {
242
-
t.Errorf("invalid error. want:nil got:%q", got.Error)
237
+
if done.CompareAndSwap(false, true) {
238
+
w.WriteHeader(http.StatusTeapot)
239
+
return
243
240
}
244
-
if attempts != 1 {
245
-
t.Errorf("invalid number of attempts. want:1 got %d", attempts)
241
+
if _, err := io.WriteString(w, "42"); err != nil {
242
+
t.Errorf("failed to write response: %v", err)
246
243
}
247
-
if got.URL != s.URL {
248
-
t.Errorf("invalid URL. want:%s got:%s", s.URL, got.URL)
249
-
}
250
-
if got.DownloadedFileBytes != 2 {
251
-
t.Errorf("invalid DownloadedFileBytes. want:2 got:%d", got.DownloadedFileBytes)
252
-
}
253
-
if got.FileSizeBytes != 2 {
254
-
t.Errorf("invalid FileSizeBytes. want:2 got:%d", got.FileSizeBytes)
255
-
}
256
-
b, err := os.ReadFile(got.DownloadedFilePath)
257
-
if err != nil {
258
-
t.Errorf("error reading downloaded file (%s): %q", got.DownloadedFilePath, err)
259
-
}
260
-
if string(b) != "42" {
261
-
t.Errorf("invalid downloaded file content. want:42 got:%s", string(b))
262
-
}
263
-
if _, ok := <-ch; ok {
264
-
t.Error("expected channel closed, but did not get it")
265
-
}
266
-
})
244
+
},
245
+
))
246
+
defer s.Close()
247
+
248
+
d := Downloader{
249
+
OutputDir: t.TempDir(),
250
+
Timeout: timeout,
251
+
MaxRetries: 4,
252
+
ConcurrencyPerServer: 1,
253
+
ChunkSize: 1024,
254
+
WaitRetry: 1 * time.Millisecond,
255
+
}
256
+
ch := d.Download(s.URL)
257
+
<-ch // discard the first status (just the file size)
258
+
got := <-ch
259
+
if got.Error != nil {
260
+
t.Errorf("invalid error. want:nil got:%q", got.Error)
261
+
}
262
+
if !done.Load() {
263
+
t.Error("expected server to be done, it is not")
264
+
}
265
+
if got.URL != s.URL {
266
+
t.Errorf("invalid URL. want:%s got:%s", s.URL, got.URL)
267
+
}
268
+
if got.DownloadedFileBytes != 2 {
269
+
t.Errorf("invalid DownloadedFileBytes. want:2 got:%d", got.DownloadedFileBytes)
270
+
}
271
+
if got.FileSizeBytes != 2 {
272
+
t.Errorf("invalid FileSizeBytes. want:2 got:%d", got.FileSizeBytes)
273
+
}
274
+
b, err := os.ReadFile(got.DownloadedFilePath)
275
+
if err != nil {
276
+
t.Errorf("error reading downloaded file (%s): %q", got.DownloadedFilePath, err)
277
+
}
278
+
if string(b) != "42" {
279
+
t.Errorf("invalid downloaded file content. want:42 got:%s", string(b))
280
+
}
281
+
if _, ok := <-ch; ok {
282
+
t.Error("expected channel closed, but did not get it")
267
283
}
268
284
}
269
285
270
286
func TestDownload_ReportPreviouslyDownloadedBytes(t *testing.T) {
287
+
t.Parallel()
271
288
tmp := t.TempDir()
272
-
firstChunk := func(url string) DownloadStatus {
273
-
d := Downloader{
274
-
OutputDir: tmp,
275
-
MaxRetries: 1,
276
-
ConcurrencyPerServer: 1,
277
-
ChunkSize: 3,
278
-
}
279
-
ch := d.Download(url)
280
-
<-ch // discard the first status (just the file size)
281
-
return <-ch
282
-
}
289
+
pdir := t.TempDir()
283
290
284
-
// first download attempt (with server up)
285
-
url, first := func() (string, DownloadStatus) {
286
-
s := httptest.NewServer(http.HandlerFunc(
287
-
func(w http.ResponseWriter, r *http.Request) {
291
+
// server that serves first chunk but always fails on second chunk
292
+
s := httptest.NewServer(http.HandlerFunc(
293
+
func(w http.ResponseWriter, r *http.Request) {
294
+
if r.Method == http.MethodHead {
295
+
w.Header().Add("Content-Length", "4")
296
+
return
297
+
}
298
+
rangeHeader := r.Header.Get("Range")
299
+
if strings.HasPrefix(rangeHeader, "bytes=0-") {
300
+
w.WriteHeader(http.StatusPartialContent)
288
301
if _, err := io.WriteString(w, "42"); err != nil {
289
302
t.Errorf("failed to write response: %v", err)
290
303
}
291
-
},
292
-
))
293
-
defer s.Close()
294
-
got := firstChunk(s.URL)
295
-
return s.URL, got
296
-
}()
304
+
return
305
+
}
306
+
w.WriteHeader(http.StatusTeapot)
307
+
},
308
+
))
309
+
defer s.Close()
297
310
298
-
// second download attempt (with server down)
299
-
second := firstChunk(url)
311
+
// first download attempt: first chunk succeeds, second chunk always fails
312
+
d := Downloader{
313
+
OutputDir: tmp,
314
+
ProgressDir: pdir,
315
+
Timeout: timeout,
316
+
MaxRetries: 1,
317
+
ConcurrencyPerServer: 1,
318
+
ChunkSize: 2,
319
+
}
320
+
ch := d.Download(s.URL)
321
+
var first DownloadStatus
322
+
for s := range ch {
323
+
first = s
324
+
}
325
+
326
+
// second download attempt: should report the previously downloaded bytes
327
+
d = Downloader{
328
+
OutputDir: tmp,
329
+
ProgressDir: pdir,
330
+
Timeout: timeout,
331
+
MaxRetries: 1,
332
+
ConcurrencyPerServer: 1,
333
+
ChunkSize: 2,
334
+
}
335
+
ch = d.Download(s.URL)
336
+
var second DownloadStatus
337
+
for s := range ch {
338
+
second = s
339
+
}
300
340
if first.DownloadedFileBytes != second.DownloadedFileBytes {
301
341
t.Errorf("expected the same number of downloaded bytes, got %d and %d", first.DownloadedFileBytes, second.DownloadedFileBytes)
302
342
}
303
343
}
304
344
305
345
func TestDownloadWithContext_ErrorUserTimeout(t *testing.T) {
306
-
userTimeout := 250 * time.Millisecond // please note that the user timeout is less than the timeout per chunk.
307
-
timeout := 10 * userTimeout
346
+
t.Parallel()
347
+
usrTimeout := 250 * time.Millisecond // smaller than overall timeout
348
+
chunkTimeout := 10 * usrTimeout
308
349
s := httptest.NewServer(http.HandlerFunc(
309
350
func(w http.ResponseWriter, r *http.Request) {
310
351
if r.Method == http.MethodHead {
311
352
w.Header().Add("Content-Length", "2")
312
353
return
313
354
}
314
-
time.Sleep(2 * userTimeout) // this time is greater than the user timeout, but shorter than the timeout per chunk.
355
+
time.Sleep(2 * usrTimeout) // greater than the user timeout, but shorter than the timeout per chunk.
315
356
},
316
357
))
317
358
defer s.Close()
318
359
d := Downloader{
319
360
OutputDir: t.TempDir(),
320
-
Timeout: timeout,
361
+
Timeout: chunkTimeout,
321
362
MaxRetries: 4,
322
363
ConcurrencyPerServer: 1,
323
364
ChunkSize: 1024,
324
365
WaitRetry: 0 * time.Second,
325
366
}
326
-
userCtx, cancFunc := context.WithTimeout(context.Background(), userTimeout)
367
+
userCtx, cancFunc := context.WithTimeout(context.Background(), usrTimeout)
327
368
defer cancFunc()
328
369
329
370
ch := d.DownloadWithContext(userCtx, s.URL)
···
332
373
if got.Error == nil {
333
374
t.Error("expected an error, but got nil")
334
375
}
335
-
if !strings.Contains(got.Error.Error(), "#4") {
336
-
t.Error("expected #4 (configured number of retries), but did not get it")
337
-
}
338
376
if _, ok := <-ch; ok {
339
377
t.Error("expected channel closed, but did not get it")
340
378
}
341
379
}
342
380
343
381
func TestDownload_Chunks(t *testing.T) {
382
+
t.Parallel()
344
383
d := DefaultDownloader()
345
384
d.ChunkSize = 5
346
385
got := d.chunks(12)
···
367
406
}
368
407
369
408
func TestGetDownload_WithUserAgent(t *testing.T) {
409
+
t.Parallel()
370
410
ua := "Answer/42.0"
371
411
s := httptest.NewServer(http.HandlerFunc(
372
412
func(w http.ResponseWriter, r *http.Request) {
···
382
422
}
383
423
384
424
func TestGetDownloadSize_ContentLength(t *testing.T) {
425
+
t.Parallel()
385
426
s := httptest.NewServer(http.HandlerFunc(
386
427
func(w http.ResponseWriter, r *http.Request) {
387
428
if _, err := io.WriteString(w, "Test"); err != nil {
···
403
444
}
404
445
405
446
func TestGetDownloadSize_WithRetry(t *testing.T) {
447
+
t.Parallel()
406
448
attempts := int32(0)
407
449
s := httptest.NewServer(http.HandlerFunc(
408
450
func(w http.ResponseWriter, r *http.Request) {
···
429
471
}
430
472
431
473
func TestGetDownloadSize_ContentRange(t *testing.T) {
474
+
t.Parallel()
432
475
s := httptest.NewServer(http.HandlerFunc(
433
476
func(w http.ResponseWriter, r *http.Request) {
434
477
w.Header().Set("Content-Range", "bytes 1-10/123")
···
451
494
}
452
495
453
496
func TestGetDownloadSize_ErrorInvalidURL(t *testing.T) {
454
-
d := DefaultDownloader()
497
+
t.Parallel()
498
+
d := Downloader{
499
+
MaxRetries: 1,
500
+
WaitRetry: 0,
501
+
Client: http.DefaultClient,
502
+
}
455
503
got, err := d.getDownloadSize(context.Background(), "test")
456
504
457
505
if err == nil {
···
463
511
}
464
512
465
513
func TestGetDownloadSize_NoContent(t *testing.T) {
514
+
t.Parallel()
466
515
s := httptest.NewServer(http.HandlerFunc(
467
516
func(w http.ResponseWriter, r *http.Request) {
468
517
if _, err := io.WriteString(w, ""); err != nil {
+1
-4
go.mod
+1
-4
go.mod
-9
go.sum
-9
go.sum
···
1
-
github.com/avast/retry-go/v4 v4.7.0 h1:yjDs35SlGvKwRNSykujfjdMxMhMQQM0TnIjJaHB+Zio=
2
-
github.com/avast/retry-go/v4 v4.7.0/go.mod h1:ZMPDa3sY2bKgpLtap9JRUgk2yTAba7cgiFhqxY2Sg6Q=
3
1
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
4
-
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
5
-
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6
2
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
7
3
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
8
-
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
9
-
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
10
4
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
11
5
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
12
6
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
13
7
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
14
8
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
15
-
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
16
-
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
17
9
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
18
-
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
19
10
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=