ai cooking
lets retry brightdata proxy erros at least (#438)
* lets retry brightdata proxy erros at least
* use external retry
* retry transport failures
---------
Co-authored-by: paul miller <paul.miller>
authored by
Paul Miller
paul miller
and committed by
GitHub
2d5fcaa0
de84eeb2
+300
-6
+2
go.mod
+2
go.mod
···
31
31
github.com/gobwas/httphead v0.1.0 // indirect
32
32
github.com/gobwas/pool v0.2.1 // indirect
33
33
github.com/gofrs/uuid v3.3.0+incompatible // indirect
34
+
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
35
+
github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
34
36
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
35
37
github.com/woodsbury/decimal128 v1.3.0 // indirect
36
38
golang.org/x/sys v0.42.0 // indirect
+4
go.sum
+4
go.sum
···
74
74
github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
75
75
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
76
76
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
77
+
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
78
+
github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
79
+
github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
80
+
github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
77
81
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
78
82
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
79
83
github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E=
+70
internal/albertsons/query/client_test.go
+70
internal/albertsons/query/client_test.go
···
8
8
"net/url"
9
9
"strings"
10
10
"testing"
11
+
"time"
12
+
13
+
retryablehttp "github.com/hashicorp/go-retryablehttp"
11
14
)
12
15
13
16
func TestNewSearchClientRequiresSubscriptionKey(t *testing.T) {
···
180
183
}
181
184
}
182
185
186
+
func TestSearch_RetriesTransient5xx(t *testing.T) {
187
+
t.Parallel()
188
+
189
+
attempts := 0
190
+
client, err := NewSearchClient(SearchClientConfig{
191
+
BaseURL: "https://www.acmemarkets.com",
192
+
SubscriptionKey: "test-subscription-key",
193
+
Reese84Provider: func(context.Context) (string, error) { return "reese-cookie", nil },
194
+
HTTPClient: retryingHTTPClient(&http.Client{
195
+
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
196
+
attempts++
197
+
if attempts < 3 {
198
+
return &http.Response{
199
+
StatusCode: http.StatusBadGateway,
200
+
Body: io.NopCloser(strings.NewReader("temporary failure")),
201
+
Request: r,
202
+
}, nil
203
+
}
204
+
return &http.Response{
205
+
StatusCode: http.StatusOK,
206
+
Header: http.Header{
207
+
"Content-Type": []string{"application/json"},
208
+
},
209
+
Body: io.NopCloser(strings.NewReader(`{"response":{"numFound":1,"disableTracking":false,"start":0,"miscInfo":{"attributionToken":"","query":"","sort":"","filter":"","nextPageToken":""},"isExactMatch":true,"docs":[{"id":"1","name":"Apples","price":1.99}]}}`)),
210
+
Request: r,
211
+
}, nil
212
+
}),
213
+
}),
214
+
})
215
+
if err != nil {
216
+
t.Fatalf("NewSearchClient returned error: %v", err)
217
+
}
218
+
219
+
payload, err := client.Search(context.Background(), "806", Category_Vegatables, SearchOptions{})
220
+
if err != nil {
221
+
t.Fatalf("Search returned error: %v", err)
222
+
}
223
+
if got, want := attempts, 3; got != want {
224
+
t.Fatalf("unexpected attempts: got %d want %d", got, want)
225
+
}
226
+
if got, want := payload.Response.NumFound, 1; got != want {
227
+
t.Fatalf("unexpected numFound: got %d want %d", got, want)
228
+
}
229
+
}
230
+
183
231
type roundTripFunc func(*http.Request) (*http.Response, error)
184
232
185
233
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
···
192
240
t.Fatalf("unexpected %s: got %q want %q", key, got, want)
193
241
}
194
242
}
243
+
244
+
func retryingHTTPClient(base *http.Client) *http.Client {
245
+
retryClient := retryablehttp.NewClient()
246
+
retryClient.Logger = nil
247
+
retryClient.HTTPClient = base
248
+
retryClient.RetryMax = 2
249
+
retryClient.RetryWaitMin = time.Millisecond
250
+
retryClient.RetryWaitMax = time.Millisecond
251
+
retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
252
+
if ctx.Err() != nil {
253
+
return false, ctx.Err()
254
+
}
255
+
if err != nil {
256
+
return false, err
257
+
}
258
+
if resp == nil || resp.Request == nil {
259
+
return false, nil
260
+
}
261
+
return resp.Request.Method == http.MethodGet && resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode <= 599, nil
262
+
}
263
+
return retryClient.StandardClient()
264
+
}
+33
-2
internal/brightdata/proxy.go
+33
-2
internal/brightdata/proxy.go
···
1
1
package brightdata
2
2
3
3
import (
4
+
"context"
4
5
"crypto/tls"
5
6
"crypto/x509"
6
7
_ "embed"
···
10
11
"net/http"
11
12
"net/url"
12
13
"os"
14
+
15
+
retryablehttp "github.com/hashicorp/go-retryablehttp"
13
16
)
14
17
15
18
//go:embed brightdata.crt
···
46
49
func NewProxyAwareHTTPClient(cfg ProxyConfig) (*http.Client, error) {
47
50
client := &http.Client{}
48
51
if !cfg.Enabled() {
49
-
return client, nil
52
+
return withRetries(client), nil
50
53
}
51
54
52
55
rootCAs, err := proxyRootCAs()
···
65
68
transport.Proxy = http.ProxyURL(cfg.proxyURL())
66
69
transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs}
67
70
client.Transport = transport
68
-
return client, nil
71
+
return withRetries(client), nil
72
+
}
73
+
74
+
func withRetries(baseClient *http.Client) *http.Client {
75
+
retryClient := retryablehttp.NewClient()
76
+
retryClient.HTTPClient = baseClient
77
+
retryClient.Logger = slog.Default()
78
+
79
+
// Keep the library defaults for now:
80
+
// RetryMax=4, RetryWaitMin=1s, RetryWaitMax=30s, Backoff=DefaultBackoff.
81
+
// We'll tune these once we have a clearer sense of how often these retries fire.
82
+
retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
83
+
if ctx.Err() != nil {
84
+
return false, ctx.Err()
85
+
}
86
+
if err != nil {
87
+
return true, err // retry these as theya re non canceled?
88
+
}
89
+
if resp == nil || resp.Request == nil {
90
+
return false, nil
91
+
}
92
+
switch resp.Request.Method {
93
+
case http.MethodGet, http.MethodHead:
94
+
default:
95
+
return false, nil
96
+
}
97
+
return resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode <= 599, nil
98
+
}
99
+
return retryClient.StandardClient()
69
100
}
70
101
71
102
func proxyRootCAs() (*x509.CertPool, error) {
+114
-4
internal/brightdata/proxy_test.go
+114
-4
internal/brightdata/proxy_test.go
···
1
1
package brightdata
2
2
3
3
import (
4
+
"context"
4
5
"net/http"
5
6
"testing"
7
+
"time"
8
+
9
+
retryablehttp "github.com/hashicorp/go-retryablehttp"
6
10
)
7
11
8
12
func TestProxyConfigValidate_AllowsDisabled(t *testing.T) {
···
57
61
t.Fatalf("expected no client timeout, got %s", client.Timeout)
58
62
}
59
63
60
-
transport, ok := client.Transport.(*http.Transport)
64
+
retryTransport, ok := client.Transport.(*retryablehttp.RoundTripper)
65
+
if !ok {
66
+
t.Fatalf("expected *retryablehttp.RoundTripper, got %T", client.Transport)
67
+
}
68
+
69
+
transport, ok := retryTransport.Client.HTTPClient.Transport.(*http.Transport)
61
70
if !ok {
62
-
t.Fatalf("expected *http.Transport, got %T", client.Transport)
71
+
t.Fatalf("expected wrapped base *http.Transport, got %T", retryTransport.Client.HTTPClient.Transport)
63
72
}
64
73
65
74
req, err := http.NewRequest(http.MethodGet, "https://www.example.com/products", nil)
···
91
100
if client.Timeout != 0 {
92
101
t.Fatalf("expected no client timeout, got %s", client.Timeout)
93
102
}
94
-
if client.Transport != nil {
95
-
t.Fatalf("expected nil transport when proxy disabled, got %T", client.Transport)
103
+
retryTransport, ok := client.Transport.(*retryablehttp.RoundTripper)
104
+
if !ok {
105
+
t.Fatalf("expected *retryablehttp.RoundTripper when proxy disabled, got %T", client.Transport)
106
+
}
107
+
if retryTransport.Client.HTTPClient.Transport != nil {
108
+
t.Fatalf("expected default base transport via nil transport, got %T", retryTransport.Client.HTTPClient.Transport)
109
+
}
110
+
}
111
+
112
+
func TestWithRetries_OnlyRetriesGet5xx(t *testing.T) {
113
+
t.Parallel()
114
+
115
+
retryClient := withRetries(&http.Client{})
116
+
117
+
transport, ok := retryClient.Transport.(*retryablehttp.RoundTripper)
118
+
if !ok {
119
+
t.Fatalf("expected *retryablehttp.RoundTripper, got %T", retryClient.Transport)
120
+
}
121
+
122
+
tests := []struct {
123
+
name string
124
+
method string
125
+
status int
126
+
want bool
127
+
}{
128
+
{name: "get 502", method: http.MethodGet, status: http.StatusBadGateway, want: true},
129
+
{name: "head 500", method: http.MethodHead, status: http.StatusInternalServerError, want: true},
130
+
{name: "get 404", method: http.MethodGet, status: http.StatusNotFound, want: false},
131
+
{name: "post 500", method: http.MethodPost, status: http.StatusInternalServerError, want: false},
96
132
}
133
+
134
+
for _, tt := range tests {
135
+
t.Run(tt.name, func(t *testing.T) {
136
+
req, err := http.NewRequestWithContext(context.Background(), tt.method, "https://example.com", nil)
137
+
if err != nil {
138
+
t.Fatalf("NewRequestWithContext() error = %v", err)
139
+
}
140
+
resp := &http.Response{StatusCode: tt.status, Request: req}
141
+
142
+
got, err := transport.Client.CheckRetry(context.Background(), resp, nil)
143
+
if err != nil {
144
+
t.Fatalf("CheckRetry() error = %v", err)
145
+
}
146
+
if got != tt.want {
147
+
t.Fatalf("unexpected retry decision: got %v want %v", got, tt.want)
148
+
}
149
+
})
150
+
}
151
+
}
152
+
153
+
func TestWithRetries_RespectsCanceledContext(t *testing.T) {
154
+
t.Parallel()
155
+
156
+
retryClient := withRetries(&http.Client{})
157
+
transport, ok := retryClient.Transport.(*retryablehttp.RoundTripper)
158
+
if !ok {
159
+
t.Fatalf("expected *retryablehttp.RoundTripper, got %T", retryClient.Transport)
160
+
}
161
+
162
+
ctx, cancel := context.WithCancel(context.Background())
163
+
cancel()
164
+
165
+
got, err := transport.Client.CheckRetry(ctx, &http.Response{
166
+
StatusCode: http.StatusBadGateway,
167
+
Request: mustRequest(t, http.MethodGet),
168
+
}, nil)
169
+
if got {
170
+
t.Fatal("expected canceled context not to retry")
171
+
}
172
+
if err != context.Canceled {
173
+
t.Fatalf("unexpected error: got %v want %v", err, context.Canceled)
174
+
}
175
+
}
176
+
177
+
func TestWithRetries_UsesLibraryDefaults(t *testing.T) {
178
+
t.Parallel()
179
+
180
+
retryClient := withRetries(&http.Client{})
181
+
transport, ok := retryClient.Transport.(*retryablehttp.RoundTripper)
182
+
if !ok {
183
+
t.Fatalf("expected *retryablehttp.RoundTripper, got %T", retryClient.Transport)
184
+
}
185
+
186
+
if got, want := transport.Client.RetryMax, 4; got != want {
187
+
t.Fatalf("unexpected RetryMax: got %d want %d", got, want)
188
+
}
189
+
if got, want := transport.Client.RetryWaitMin, time.Second; got != want {
190
+
t.Fatalf("unexpected RetryWaitMin: got %s want %s", got, want)
191
+
}
192
+
if got, want := transport.Client.RetryWaitMax, 30*time.Second; got != want {
193
+
t.Fatalf("unexpected RetryWaitMax: got %s want %s", got, want)
194
+
}
195
+
if got := transport.Client.Backoff(time.Second, 30*time.Second, 0, nil); got != time.Second {
196
+
t.Fatalf("unexpected default backoff for attempt 0: got %s want %s", got, time.Second)
197
+
}
198
+
}
199
+
200
+
func mustRequest(t *testing.T, method string) *http.Request {
201
+
t.Helper()
202
+
req, err := http.NewRequestWithContext(context.Background(), method, "https://example.com", nil)
203
+
if err != nil {
204
+
t.Fatalf("NewRequestWithContext() error = %v", err)
205
+
}
206
+
return req
97
207
}
+77
internal/wholefoods/client_test.go
+77
internal/wholefoods/client_test.go
···
4
4
"context"
5
5
"encoding/json"
6
6
"fmt"
7
+
"io"
7
8
"net/http"
8
9
"net/http/httptest"
9
10
"os"
···
11
12
"strconv"
12
13
"strings"
13
14
"testing"
15
+
"time"
16
+
17
+
retryablehttp "github.com/hashicorp/go-retryablehttp"
14
18
)
15
19
16
20
func TestCategory_BuildsRequestAndDecodesFixture(t *testing.T) {
···
98
102
}
99
103
}
100
104
105
+
func TestCategory_RetriesTransient5xx(t *testing.T) {
106
+
t.Parallel()
107
+
108
+
attempts := 0
109
+
client := NewClientWithBaseURL("https://example.com", retryingHTTPClient(&http.Client{
110
+
Transport: wholeFoodsRoundTripFunc(func(r *http.Request) (*http.Response, error) {
111
+
attempts++
112
+
if attempts < 3 {
113
+
return &http.Response{
114
+
StatusCode: http.StatusBadGateway,
115
+
Header: make(http.Header),
116
+
Body: ioNopCloserString("temporary failure"),
117
+
Request: r,
118
+
}, nil
119
+
}
120
+
return &http.Response{
121
+
StatusCode: http.StatusOK,
122
+
Header: http.Header{
123
+
"Content-Type": []string{"application/json"},
124
+
},
125
+
Body: ioNopCloserString(`{"results":[{"name":"Retry Beef","slug":"retry-beef","brand":"Whole Foods","store":10216}],"meta":{"total":{"value":1,"relation":"eq"},"state":{"refinements":[],"sort":""}}}`),
126
+
Request: r,
127
+
}, nil
128
+
}),
129
+
}))
130
+
131
+
resp, err := client.Category(context.Background(), "beef", "10216")
132
+
if err != nil {
133
+
t.Fatalf("Category returned error: %v", err)
134
+
}
135
+
if got, want := attempts, 3; got != want {
136
+
t.Fatalf("unexpected attempts: got %d want %d", got, want)
137
+
}
138
+
if got, want := len(resp), 1; got != want {
139
+
t.Fatalf("unexpected result count: got %d want %d", got, want)
140
+
}
141
+
if got := resp[0].Name; got != "Retry Beef" {
142
+
t.Fatalf("unexpected product name: %q", got)
143
+
}
144
+
}
145
+
101
146
func TestCategory_PaginatesUntilShortPage(t *testing.T) {
102
147
t.Parallel()
103
148
···
236
281
}
237
282
return data
238
283
}
284
+
285
+
type wholeFoodsRoundTripFunc func(*http.Request) (*http.Response, error)
286
+
287
+
func (f wholeFoodsRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
288
+
return f(r)
289
+
}
290
+
291
+
func ioNopCloserString(body string) io.ReadCloser {
292
+
return io.NopCloser(strings.NewReader(body))
293
+
}
294
+
295
+
func retryingHTTPClient(base *http.Client) *http.Client {
296
+
retryClient := retryablehttp.NewClient()
297
+
retryClient.Logger = nil
298
+
retryClient.HTTPClient = base
299
+
retryClient.RetryMax = 2
300
+
retryClient.RetryWaitMin = time.Millisecond
301
+
retryClient.RetryWaitMax = time.Millisecond
302
+
retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) {
303
+
if ctx.Err() != nil {
304
+
return false, ctx.Err()
305
+
}
306
+
if err != nil {
307
+
return false, err
308
+
}
309
+
if resp == nil || resp.Request == nil {
310
+
return false, nil
311
+
}
312
+
return resp.Request.Method == http.MethodGet && resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode <= 599, nil
313
+
}
314
+
return retryClient.StandardClient()
315
+
}