🧱 Chunk is a download manager for slow and unstable servers
Merge pull request #37 from cuducos/export-http-client
Exports HTTP client
authored by
Daniel Fireman
and committed by
GitHub
69afbce8
02999cdd
+11
-8
+11
-8
downloader.go
+11
-8
downloader.go
···
58
58
// defaults to the current working directory.
59
59
OutputDir string
60
60
61
-
// client is the HTTP client used for every request needed to download all
62
-
// the files.
63
-
client *http.Client
61
+
// Client is the HTTP client used for all requests. It uses a customized
62
+
// HTTP transport and timeout to handle content ranges download and
63
+
// parallel requests to the same server. Check NewHTTPClient for
64
+
// customizing it.
65
+
Client *http.Client
64
66
65
67
// TimeoutPerChunk is the timeout for the download of each chunk from each
66
68
// URL. A chunk is a part of a file requested using the content range HTTP
···
103
105
return nil, fmt.Errorf("error creating the request for %s: %w", u, err)
104
106
}
105
107
req.Header.Set("Range", c.rangeHeader())
106
-
resp, err := d.client.Do(req)
108
+
resp, err := d.Client.Do(req)
107
109
if err != nil {
108
110
return nil, fmt.Errorf("error sending a get http request to %s: %w", u, err)
109
111
}
···
154
156
if err != nil {
155
157
return fmt.Errorf("creating the request for %s: %w", u, err)
156
158
}
157
-
resp, err := d.client.Do(req)
159
+
resp, err := d.Client.Do(req)
158
160
if err != nil {
159
161
return fmt.Errorf("dispatching the request for %s: %w", u, err)
160
162
}
···
277
279
// DownloadWithContext is a version of Download that takes a context. The
278
280
// context can be used to stop all downloads in progress.
279
281
func (d *Downloader) DownloadWithContext(ctx context.Context, urls ...string) <-chan DownloadStatus {
280
-
if d.client == nil {
281
-
d.client = newClient(d.ConcurrencyPerServer, d.Timeout)
282
+
if d.Client == nil {
283
+
d.Client = newClient(d.ConcurrencyPerServer, d.Timeout)
282
284
}
283
285
ch := make(chan DownloadStatus, 2*len(urls)) // the first status will be the total file size (and or an error creating/trucating the file).
284
286
var wg sync.WaitGroup // this wait group is used to wait for all chunks (from all downloads) to finish.
···
317
319
MaxRetries: DefaultMaxRetries,
318
320
ChunkSize: DefaultChunkSize,
319
321
WaitRetry: DefaultWaitRetry,
320
-
client: newClient(DefaultMaxRetries, DefaultTimeout),
322
+
Client: newClient(DefaultMaxRetries, DefaultTimeout),
321
323
}
322
324
}
323
325
···
325
327
t := http.DefaultTransport.(*http.Transport).Clone()
326
328
t.MaxConnsPerHost = maxParallelDownloadsPerServer
327
329
t.MaxIdleConnsPerHost = maxParallelDownloadsPerServer
330
+
328
331
return &http.Client{
329
332
Timeout: timeoutPerChunk,
330
333
Transport: t,