🧱 Chunk is a download manager for slow and unstable servers
Merge pull request #1 from cuducos/get-with-retries-and-timeout
Prototypes a GET HTTP request with retries and an independent timeout
authored by
Eduardo Cuducos
and committed by
GitHub
6862f830
384575ba
+229
+11
.github/workflows/gofmt.yaml
+11
.github/workflows/gofmt.yaml
+12
.github/workflows/golint.yaml
+12
.github/workflows/golint.yaml
+12
.github/workflows/tests.yaml
+12
.github/workflows/tests.yaml
+8
README.md
+8
README.md
···
62
62
* Read the bytes from the `results` channel
63
63
* Write to the file on disk
64
64
* Update a progress bar to give the user an idea about the status of the downloads
65
+
66
+
## Prototype
67
+
68
+
The prototype is a CLI that wraps a GET HTTP request in a 45s timeout independent of the HTTP client's timeout. It also includes 3 retries.
69
+
70
+
```console
71
+
$ go run main.go <URL> # e.g. go run main.go https://github.com/cuducos/chunk
72
+
```
+5
go.mod
+5
go.mod
+14
go.sum
+14
go.sum
···
1
+
github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
2
+
github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
3
+
github.com/avast/retry-go/v4 v4.3.0 h1:cqI48aXx0BExKoM7XPklDpoHAg7/srPPLAfWG5z62jo=
4
+
github.com/avast/retry-go/v4 v4.3.0/go.mod h1:bqOlT4nxk4phk9buiQFaghzjpqdchOSwPgjdfdQBtdg=
5
+
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6
+
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
7
+
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
8
+
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
9
+
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
10
+
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
11
+
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
12
+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
13
+
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
14
+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+99
main.go
+99
main.go
···
1
+
package main
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"fmt"
7
+
"log"
8
+
"net/http"
9
+
"os"
10
+
"time"
11
+
12
+
"github.com/avast/retry-go"
13
+
)
14
+
15
+
const (
16
+
defaultRetries = 3
17
+
defaultTimeout = 45 * time.Second
18
+
)
19
+
20
+
type downloader struct {
21
+
client *http.Client
22
+
retries uint
23
+
}
24
+
25
+
func (d *downloader) downloadWithContext(ctx context.Context, u string) ([]byte, error) {
26
+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
27
+
if err != nil {
28
+
return nil, fmt.Errorf("error creating the request for %s: %w", u, err)
29
+
}
30
+
req = req.WithContext(ctx)
31
+
resp, err := d.client.Do(req)
32
+
if err != nil {
33
+
return nil, fmt.Errorf("error sending a get http request to %s: %w", u, err)
34
+
}
35
+
defer resp.Body.Close()
36
+
if resp.StatusCode != http.StatusOK {
37
+
return nil, fmt.Errorf("got http response %s from %s: %w", resp.Status, u, err)
38
+
}
39
+
var b bytes.Buffer
40
+
_, err = b.ReadFrom(resp.Body)
41
+
if err != nil {
42
+
return nil, fmt.Errorf("error reading body from %s: %w", u, err)
43
+
}
44
+
return b.Bytes(), nil
45
+
}
46
+
47
+
func (d *downloader) downloadWithTimeout(u string) ([]byte, error) {
48
+
ctx, cancel := context.WithTimeout(context.Background(), d.client.Timeout)
49
+
defer cancel()
50
+
ch := make(chan []byte)
51
+
errs := make(chan error)
52
+
go func() {
53
+
b, err := d.downloadWithContext(ctx, u)
54
+
if err != nil {
55
+
errs <- err
56
+
return
57
+
}
58
+
ch <- b
59
+
}()
60
+
select {
61
+
case <-ctx.Done():
62
+
return nil, fmt.Errorf("request to %s ended due to timeout: %w", u, ctx.Err())
63
+
case err := <-errs:
64
+
return nil, fmt.Errorf("request to %s failed: %w", u, err)
65
+
case b := <-ch:
66
+
return b, nil
67
+
}
68
+
}
69
+
70
+
func (d *downloader) download(u string) ([]byte, error) {
71
+
ch := make(chan []byte, 1)
72
+
defer close(ch)
73
+
err := retry.Do(
74
+
func() error {
75
+
b, err := d.downloadWithTimeout(u)
76
+
if err != nil {
77
+
return err
78
+
}
79
+
ch <- b
80
+
return nil
81
+
},
82
+
retry.Attempts(d.retries),
83
+
retry.MaxDelay(d.client.Timeout),
84
+
)
85
+
if err != nil {
86
+
return nil, fmt.Errorf("error downloading %s: %w", u, err)
87
+
}
88
+
b := <-ch
89
+
return b, nil
90
+
}
91
+
92
+
func main() {
93
+
d := downloader{&http.Client{Timeout: defaultTimeout}, uint(defaultRetries)}
94
+
b, err := d.download(os.Args[1])
95
+
if err != nil {
96
+
log.Fatal(err)
97
+
}
98
+
fmt.Print(string(b))
99
+
}
+68
main_test.go
+68
main_test.go
···
1
+
package main
2
+
3
+
import (
4
+
"fmt"
5
+
"net/http"
6
+
"net/http/httptest"
7
+
"sync/atomic"
8
+
"testing"
9
+
"time"
10
+
)
11
+
12
+
func testServer(t *testing.T) *httptest.Server {
13
+
var attempt int32
14
+
paths := make(map[string]func(http.ResponseWriter))
15
+
paths["/ok"] = func(w http.ResponseWriter) {
16
+
fmt.Fprintf(w, "42")
17
+
}
18
+
paths["/retry"] = func(w http.ResponseWriter) {
19
+
if atomic.LoadInt32(&attempt) == 0 {
20
+
atomic.StoreInt32(&attempt, 1)
21
+
time.Sleep(1 * time.Second)
22
+
}
23
+
fmt.Fprintf(w, "42")
24
+
}
25
+
paths["/slow"] = func(w http.ResponseWriter) {
26
+
time.Sleep(1 * time.Second)
27
+
}
28
+
29
+
return httptest.NewServer(
30
+
http.HandlerFunc(
31
+
func(w http.ResponseWriter, r *http.Request) {
32
+
h, ok := paths[r.URL.Path]
33
+
if !ok {
34
+
t.Fatalf("unknown url path for the test server %s", r.URL.Path)
35
+
}
36
+
h(w)
37
+
},
38
+
),
39
+
)
40
+
}
41
+
42
+
func TestGet(t *testing.T) {
43
+
s := testServer(t)
44
+
defer s.Close()
45
+
for _, tc := range []struct {
46
+
desc string
47
+
path string
48
+
expected []byte
49
+
}{
50
+
{"normal response", "/ok", []byte("42")},
51
+
{"retried response", "/retry", []byte("42")},
52
+
{"timeout", "/slow", nil},
53
+
} {
54
+
t.Run(tc.desc, func(t *testing.T) {
55
+
d := downloader{&http.Client{Timeout: 250 * time.Millisecond}, 3}
56
+
got, err := d.download(s.URL + tc.path)
57
+
if string(got) != string(tc.expected) {
58
+
t.Errorf("expected %s, got %s", string(tc.expected), string(got))
59
+
}
60
+
if tc.expected == nil && err == nil {
61
+
t.Error("expected an error, but got nil")
62
+
}
63
+
if tc.expected != nil && err != nil {
64
+
t.Errorf("expected no error, but got %s", err)
65
+
}
66
+
})
67
+
}
68
+
}