🧱 Chunk is a download manager for slow and unstable servers
Adds get file size method
Closes #3
Co-authored by: Eduardo Cuducos <cuducos@users.noreply.github.com>
Co-authored by: Daniel Fireman <danielfireman@users.noreply.github.com>
authored by
Vinnie Mesel
and committed by
GitHub
347ff2b4
75aa3fc5
+54
-9
+32
-3
main.go
+32
-3
main.go
···
8
8
"net/http"
9
9
"os"
10
10
"path/filepath"
11
+
"strings"
11
12
"sync"
12
13
"time"
13
14
···
133
134
}
134
135
}
135
136
137
+
func (d *Downloader) getDownloadSize(ctx context.Context, u string) (uint64, error) {
138
+
req, err := http.NewRequestWithContext(ctx, http.MethodHead, u, nil)
139
+
if err != nil {
140
+
return 0, fmt.Errorf("creating the request for %s: %w", u, err)
141
+
}
142
+
resp, err := d.client.Do(req)
143
+
if err != nil {
144
+
return 0, fmt.Errorf("sending get http request to %s: %w", u, err)
145
+
}
146
+
defer resp.Body.Close()
147
+
if resp.StatusCode != 200 {
148
+
return 0, fmt.Errorf("got unexpected http response status for %s: %s", u, resp.Status)
149
+
}
150
+
if resp.ContentLength <= 0 && resp.Header.Get("Content-Range") != "" {
151
+
var s uint64
152
+
p := strings.Split(resp.Header.Get("Content-Range"), "/")
153
+
fmt.Sscan(p[len(p)-1], &s)
154
+
return s, nil
155
+
}
156
+
return uint64(resp.ContentLength), nil
157
+
}
158
+
136
159
func (d *Downloader) downloadFile(ctx context.Context, u string) ([]byte, error) {
137
160
ch := make(chan []byte, 1)
138
161
defer close(ch)
···
193
216
wg.Add(1)
194
217
go func(u string) {
195
218
defer wg.Done()
196
-
s := DownloadStatus{URL: u}
219
+
path := filepath.Join(os.TempDir(), filepath.Base(u))
220
+
s := DownloadStatus{URL: u, DownloadedFilePath: path}
197
221
defer func() { ch <- s }()
198
-
s.DownloadedFilePath = filepath.Join(os.TempDir(), filepath.Base(u))
222
+
t, err := d.getDownloadSize(ctx, u) // TODO: retry
223
+
if err != nil {
224
+
s.Error = fmt.Errorf("error getting file size: %w", err)
225
+
return
226
+
}
227
+
s.FileSizeBytes = t
228
+
ch <- s // send total file size to the user
199
229
b, err := d.downloadFile(ctx, u)
200
230
if err != nil {
201
231
s.Error = err
···
206
236
return
207
237
}
208
238
s.DownloadedFileBytes = uint64(len(b))
209
-
s.FileSizeBytes = uint64(len(b))
210
239
}(u)
211
240
}
212
241
go func() {
+22
-6
main_test.go
+22
-6
main_test.go
···
24
24
t.Run(tc.desc, func(t *testing.T) {
25
25
s := httptest.NewServer(http.HandlerFunc(
26
26
func(w http.ResponseWriter, r *http.Request) {
27
+
if r.Method == http.MethodHead {
28
+
return
29
+
}
27
30
tc.proc(w)
28
31
},
29
32
))
···
36
39
WaitBetweenRetries: 0 * time.Second,
37
40
}
38
41
ch := d.Download(s.URL)
39
-
status := <-ch
40
-
if status.Error == nil {
42
+
<-ch // discard the first got (just the file size)
43
+
got := <-ch
44
+
if got.Error == nil {
41
45
t.Error("expected an error, but got nil")
42
46
}
43
-
if !strings.Contains(status.Error.Error(), "#4") {
47
+
if !strings.Contains(got.Error.Error(), "#4") {
44
48
t.Error("expected #4 (configured number of retries), but did not get it")
45
49
}
46
50
if _, ok := <-ch; ok {
···
59
63
defer s.Close()
60
64
61
65
ch := DefaultDownloader().Download(s.URL)
66
+
<-ch // discard the first status (just the file size)
62
67
got := <-ch
63
68
defer os.Remove(got.DownloadedFilePath)
64
69
···
99
104
attempts := int32(0)
100
105
s := httptest.NewServer(http.HandlerFunc(
101
106
func(w http.ResponseWriter, r *http.Request) {
107
+
if r.Method == http.MethodHead {
108
+
w.Header().Add("Content-Length", "2")
109
+
return
110
+
}
102
111
if atomic.CompareAndSwapInt32(&attempts, 0, 1) {
103
112
tc.proc(w)
104
113
}
···
115
124
WaitBetweenRetries: 0 * time.Second,
116
125
}
117
126
ch := d.Download(s.URL)
127
+
<-ch // discard the first status (just the file size)
118
128
got := <-ch
119
129
if got.Error != nil {
120
130
t.Errorf("invalid error. want:nil got:%q", got.Error)
···
150
160
timeout := 10 * userTimeout
151
161
s := httptest.NewServer(http.HandlerFunc(
152
162
func(w http.ResponseWriter, r *http.Request) {
163
+
if r.Method == http.MethodHead {
164
+
return
165
+
}
153
166
time.Sleep(2 * userTimeout) // this time is greater than the user timeout, but shorter than the timeout per chunk.
154
167
},
155
168
))
···
165
178
defer cancFunc()
166
179
167
180
ch := d.DownloadWithContext(userCtx, s.URL)
168
-
status := <-ch
169
-
if status.Error == nil {
181
+
<-ch // discard the first got (just the file size)
182
+
got := <-ch
183
+
if got.Error == nil {
170
184
t.Error("expected an error, but got nil")
171
185
}
172
-
if !strings.Contains(status.Error.Error(), "#4") {
186
+
if !strings.Contains(got.Error.Error(), "#4") {
173
187
t.Error("expected #4 (configured number of retries), but did not get it")
174
188
}
175
189
if _, ok := <-ch; ok {
···
202
216
}
203
217
}
204
218
}
219
+
220
+
// TODO: add tests for getDownloadSize (success with Content-Length, success with Content-Range, failure)