ai cooking
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

lets retry brightdata proxy erros at least (#438)

* lets retry brightdata proxy erros at least

* use external retry

* retry transport failures

---------

Co-authored-by: paul miller <paul.miller>

authored by

Paul Miller
paul miller
and committed by
GitHub
2d5fcaa0 de84eeb2

+300 -6
+2
go.mod
··· 31 31 github.com/gobwas/httphead v0.1.0 // indirect 32 32 github.com/gobwas/pool v0.2.1 // indirect 33 33 github.com/gofrs/uuid v3.3.0+incompatible // indirect 34 + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 35 + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect 34 36 github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect 35 37 github.com/woodsbury/decimal128 v1.3.0 // indirect 36 38 golang.org/x/sys v0.42.0 // indirect
+4
go.sum
··· 74 74 github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 75 75 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 76 76 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 77 + github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= 78 + github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= 79 + github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= 80 + github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= 77 81 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 78 82 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 79 83 github.com/invopop/jsonschema v0.13.0 h1:KvpoAJWEjR3uD9Kbm2HWJmqsEaHt8lBUpd0qHcIi21E=
+70
internal/albertsons/query/client_test.go
··· 8 8 "net/url" 9 9 "strings" 10 10 "testing" 11 + "time" 12 + 13 + retryablehttp "github.com/hashicorp/go-retryablehttp" 11 14 ) 12 15 13 16 func TestNewSearchClientRequiresSubscriptionKey(t *testing.T) { ··· 180 183 } 181 184 } 182 185 186 + func TestSearch_RetriesTransient5xx(t *testing.T) { 187 + t.Parallel() 188 + 189 + attempts := 0 190 + client, err := NewSearchClient(SearchClientConfig{ 191 + BaseURL: "https://www.acmemarkets.com", 192 + SubscriptionKey: "test-subscription-key", 193 + Reese84Provider: func(context.Context) (string, error) { return "reese-cookie", nil }, 194 + HTTPClient: retryingHTTPClient(&http.Client{ 195 + Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { 196 + attempts++ 197 + if attempts < 3 { 198 + return &http.Response{ 199 + StatusCode: http.StatusBadGateway, 200 + Body: io.NopCloser(strings.NewReader("temporary failure")), 201 + Request: r, 202 + }, nil 203 + } 204 + return &http.Response{ 205 + StatusCode: http.StatusOK, 206 + Header: http.Header{ 207 + "Content-Type": []string{"application/json"}, 208 + }, 209 + Body: io.NopCloser(strings.NewReader(`{"response":{"numFound":1,"disableTracking":false,"start":0,"miscInfo":{"attributionToken":"","query":"","sort":"","filter":"","nextPageToken":""},"isExactMatch":true,"docs":[{"id":"1","name":"Apples","price":1.99}]}}`)), 210 + Request: r, 211 + }, nil 212 + }), 213 + }), 214 + }) 215 + if err != nil { 216 + t.Fatalf("NewSearchClient returned error: %v", err) 217 + } 218 + 219 + payload, err := client.Search(context.Background(), "806", Category_Vegatables, SearchOptions{}) 220 + if err != nil { 221 + t.Fatalf("Search returned error: %v", err) 222 + } 223 + if got, want := attempts, 3; got != want { 224 + t.Fatalf("unexpected attempts: got %d want %d", got, want) 225 + } 226 + if got, want := payload.Response.NumFound, 1; got != want { 227 + t.Fatalf("unexpected numFound: got %d want %d", got, want) 228 + } 229 + } 230 + 183 231 type roundTripFunc func(*http.Request) (*http.Response, error) 184 232 185 233 func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { ··· 192 240 t.Fatalf("unexpected %s: got %q want %q", key, got, want) 193 241 } 194 242 } 243 + 244 + func retryingHTTPClient(base *http.Client) *http.Client { 245 + retryClient := retryablehttp.NewClient() 246 + retryClient.Logger = nil 247 + retryClient.HTTPClient = base 248 + retryClient.RetryMax = 2 249 + retryClient.RetryWaitMin = time.Millisecond 250 + retryClient.RetryWaitMax = time.Millisecond 251 + retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { 252 + if ctx.Err() != nil { 253 + return false, ctx.Err() 254 + } 255 + if err != nil { 256 + return false, err 257 + } 258 + if resp == nil || resp.Request == nil { 259 + return false, nil 260 + } 261 + return resp.Request.Method == http.MethodGet && resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode <= 599, nil 262 + } 263 + return retryClient.StandardClient() 264 + }
+33 -2
internal/brightdata/proxy.go
··· 1 1 package brightdata 2 2 3 3 import ( 4 + "context" 4 5 "crypto/tls" 5 6 "crypto/x509" 6 7 _ "embed" ··· 10 11 "net/http" 11 12 "net/url" 12 13 "os" 14 + 15 + retryablehttp "github.com/hashicorp/go-retryablehttp" 13 16 ) 14 17 15 18 //go:embed brightdata.crt ··· 46 49 func NewProxyAwareHTTPClient(cfg ProxyConfig) (*http.Client, error) { 47 50 client := &http.Client{} 48 51 if !cfg.Enabled() { 49 - return client, nil 52 + return withRetries(client), nil 50 53 } 51 54 52 55 rootCAs, err := proxyRootCAs() ··· 65 68 transport.Proxy = http.ProxyURL(cfg.proxyURL()) 66 69 transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs} 67 70 client.Transport = transport 68 - return client, nil 71 + return withRetries(client), nil 72 + } 73 + 74 + func withRetries(baseClient *http.Client) *http.Client { 75 + retryClient := retryablehttp.NewClient() 76 + retryClient.HTTPClient = baseClient 77 + retryClient.Logger = slog.Default() 78 + 79 + // Keep the library defaults for now: 80 + // RetryMax=4, RetryWaitMin=1s, RetryWaitMax=30s, Backoff=DefaultBackoff. 81 + // We'll tune these once we have a clearer sense of how often these retries fire. 82 + retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { 83 + if ctx.Err() != nil { 84 + return false, ctx.Err() 85 + } 86 + if err != nil { 87 + return true, err // retry these as theya re non canceled? 88 + } 89 + if resp == nil || resp.Request == nil { 90 + return false, nil 91 + } 92 + switch resp.Request.Method { 93 + case http.MethodGet, http.MethodHead: 94 + default: 95 + return false, nil 96 + } 97 + return resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode <= 599, nil 98 + } 99 + return retryClient.StandardClient() 69 100 } 70 101 71 102 func proxyRootCAs() (*x509.CertPool, error) {
+114 -4
internal/brightdata/proxy_test.go
··· 1 1 package brightdata 2 2 3 3 import ( 4 + "context" 4 5 "net/http" 5 6 "testing" 7 + "time" 8 + 9 + retryablehttp "github.com/hashicorp/go-retryablehttp" 6 10 ) 7 11 8 12 func TestProxyConfigValidate_AllowsDisabled(t *testing.T) { ··· 57 61 t.Fatalf("expected no client timeout, got %s", client.Timeout) 58 62 } 59 63 60 - transport, ok := client.Transport.(*http.Transport) 64 + retryTransport, ok := client.Transport.(*retryablehttp.RoundTripper) 65 + if !ok { 66 + t.Fatalf("expected *retryablehttp.RoundTripper, got %T", client.Transport) 67 + } 68 + 69 + transport, ok := retryTransport.Client.HTTPClient.Transport.(*http.Transport) 61 70 if !ok { 62 - t.Fatalf("expected *http.Transport, got %T", client.Transport) 71 + t.Fatalf("expected wrapped base *http.Transport, got %T", retryTransport.Client.HTTPClient.Transport) 63 72 } 64 73 65 74 req, err := http.NewRequest(http.MethodGet, "https://www.example.com/products", nil) ··· 91 100 if client.Timeout != 0 { 92 101 t.Fatalf("expected no client timeout, got %s", client.Timeout) 93 102 } 94 - if client.Transport != nil { 95 - t.Fatalf("expected nil transport when proxy disabled, got %T", client.Transport) 103 + retryTransport, ok := client.Transport.(*retryablehttp.RoundTripper) 104 + if !ok { 105 + t.Fatalf("expected *retryablehttp.RoundTripper when proxy disabled, got %T", client.Transport) 106 + } 107 + if retryTransport.Client.HTTPClient.Transport != nil { 108 + t.Fatalf("expected default base transport via nil transport, got %T", retryTransport.Client.HTTPClient.Transport) 109 + } 110 + } 111 + 112 + func TestWithRetries_OnlyRetriesGet5xx(t *testing.T) { 113 + t.Parallel() 114 + 115 + retryClient := withRetries(&http.Client{}) 116 + 117 + transport, ok := retryClient.Transport.(*retryablehttp.RoundTripper) 118 + if !ok { 119 + t.Fatalf("expected *retryablehttp.RoundTripper, got %T", retryClient.Transport) 120 + } 121 + 122 + tests := []struct { 123 + name string 124 + method string 125 + status int 126 + want bool 127 + }{ 128 + {name: "get 502", method: http.MethodGet, status: http.StatusBadGateway, want: true}, 129 + {name: "head 500", method: http.MethodHead, status: http.StatusInternalServerError, want: true}, 130 + {name: "get 404", method: http.MethodGet, status: http.StatusNotFound, want: false}, 131 + {name: "post 500", method: http.MethodPost, status: http.StatusInternalServerError, want: false}, 96 132 } 133 + 134 + for _, tt := range tests { 135 + t.Run(tt.name, func(t *testing.T) { 136 + req, err := http.NewRequestWithContext(context.Background(), tt.method, "https://example.com", nil) 137 + if err != nil { 138 + t.Fatalf("NewRequestWithContext() error = %v", err) 139 + } 140 + resp := &http.Response{StatusCode: tt.status, Request: req} 141 + 142 + got, err := transport.Client.CheckRetry(context.Background(), resp, nil) 143 + if err != nil { 144 + t.Fatalf("CheckRetry() error = %v", err) 145 + } 146 + if got != tt.want { 147 + t.Fatalf("unexpected retry decision: got %v want %v", got, tt.want) 148 + } 149 + }) 150 + } 151 + } 152 + 153 + func TestWithRetries_RespectsCanceledContext(t *testing.T) { 154 + t.Parallel() 155 + 156 + retryClient := withRetries(&http.Client{}) 157 + transport, ok := retryClient.Transport.(*retryablehttp.RoundTripper) 158 + if !ok { 159 + t.Fatalf("expected *retryablehttp.RoundTripper, got %T", retryClient.Transport) 160 + } 161 + 162 + ctx, cancel := context.WithCancel(context.Background()) 163 + cancel() 164 + 165 + got, err := transport.Client.CheckRetry(ctx, &http.Response{ 166 + StatusCode: http.StatusBadGateway, 167 + Request: mustRequest(t, http.MethodGet), 168 + }, nil) 169 + if got { 170 + t.Fatal("expected canceled context not to retry") 171 + } 172 + if err != context.Canceled { 173 + t.Fatalf("unexpected error: got %v want %v", err, context.Canceled) 174 + } 175 + } 176 + 177 + func TestWithRetries_UsesLibraryDefaults(t *testing.T) { 178 + t.Parallel() 179 + 180 + retryClient := withRetries(&http.Client{}) 181 + transport, ok := retryClient.Transport.(*retryablehttp.RoundTripper) 182 + if !ok { 183 + t.Fatalf("expected *retryablehttp.RoundTripper, got %T", retryClient.Transport) 184 + } 185 + 186 + if got, want := transport.Client.RetryMax, 4; got != want { 187 + t.Fatalf("unexpected RetryMax: got %d want %d", got, want) 188 + } 189 + if got, want := transport.Client.RetryWaitMin, time.Second; got != want { 190 + t.Fatalf("unexpected RetryWaitMin: got %s want %s", got, want) 191 + } 192 + if got, want := transport.Client.RetryWaitMax, 30*time.Second; got != want { 193 + t.Fatalf("unexpected RetryWaitMax: got %s want %s", got, want) 194 + } 195 + if got := transport.Client.Backoff(time.Second, 30*time.Second, 0, nil); got != time.Second { 196 + t.Fatalf("unexpected default backoff for attempt 0: got %s want %s", got, time.Second) 197 + } 198 + } 199 + 200 + func mustRequest(t *testing.T, method string) *http.Request { 201 + t.Helper() 202 + req, err := http.NewRequestWithContext(context.Background(), method, "https://example.com", nil) 203 + if err != nil { 204 + t.Fatalf("NewRequestWithContext() error = %v", err) 205 + } 206 + return req 97 207 }
+77
internal/wholefoods/client_test.go
··· 4 4 "context" 5 5 "encoding/json" 6 6 "fmt" 7 + "io" 7 8 "net/http" 8 9 "net/http/httptest" 9 10 "os" ··· 11 12 "strconv" 12 13 "strings" 13 14 "testing" 15 + "time" 16 + 17 + retryablehttp "github.com/hashicorp/go-retryablehttp" 14 18 ) 15 19 16 20 func TestCategory_BuildsRequestAndDecodesFixture(t *testing.T) { ··· 98 102 } 99 103 } 100 104 105 + func TestCategory_RetriesTransient5xx(t *testing.T) { 106 + t.Parallel() 107 + 108 + attempts := 0 109 + client := NewClientWithBaseURL("https://example.com", retryingHTTPClient(&http.Client{ 110 + Transport: wholeFoodsRoundTripFunc(func(r *http.Request) (*http.Response, error) { 111 + attempts++ 112 + if attempts < 3 { 113 + return &http.Response{ 114 + StatusCode: http.StatusBadGateway, 115 + Header: make(http.Header), 116 + Body: ioNopCloserString("temporary failure"), 117 + Request: r, 118 + }, nil 119 + } 120 + return &http.Response{ 121 + StatusCode: http.StatusOK, 122 + Header: http.Header{ 123 + "Content-Type": []string{"application/json"}, 124 + }, 125 + Body: ioNopCloserString(`{"results":[{"name":"Retry Beef","slug":"retry-beef","brand":"Whole Foods","store":10216}],"meta":{"total":{"value":1,"relation":"eq"},"state":{"refinements":[],"sort":""}}}`), 126 + Request: r, 127 + }, nil 128 + }), 129 + })) 130 + 131 + resp, err := client.Category(context.Background(), "beef", "10216") 132 + if err != nil { 133 + t.Fatalf("Category returned error: %v", err) 134 + } 135 + if got, want := attempts, 3; got != want { 136 + t.Fatalf("unexpected attempts: got %d want %d", got, want) 137 + } 138 + if got, want := len(resp), 1; got != want { 139 + t.Fatalf("unexpected result count: got %d want %d", got, want) 140 + } 141 + if got := resp[0].Name; got != "Retry Beef" { 142 + t.Fatalf("unexpected product name: %q", got) 143 + } 144 + } 145 + 101 146 func TestCategory_PaginatesUntilShortPage(t *testing.T) { 102 147 t.Parallel() 103 148 ··· 236 281 } 237 282 return data 238 283 } 284 + 285 + type wholeFoodsRoundTripFunc func(*http.Request) (*http.Response, error) 286 + 287 + func (f wholeFoodsRoundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { 288 + return f(r) 289 + } 290 + 291 + func ioNopCloserString(body string) io.ReadCloser { 292 + return io.NopCloser(strings.NewReader(body)) 293 + } 294 + 295 + func retryingHTTPClient(base *http.Client) *http.Client { 296 + retryClient := retryablehttp.NewClient() 297 + retryClient.Logger = nil 298 + retryClient.HTTPClient = base 299 + retryClient.RetryMax = 2 300 + retryClient.RetryWaitMin = time.Millisecond 301 + retryClient.RetryWaitMax = time.Millisecond 302 + retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { 303 + if ctx.Err() != nil { 304 + return false, ctx.Err() 305 + } 306 + if err != nil { 307 + return false, err 308 + } 309 + if resp == nil || resp.Request == nil { 310 + return false, nil 311 + } 312 + return resp.Request.Method == http.MethodGet && resp.StatusCode >= http.StatusInternalServerError && resp.StatusCode <= 599, nil 313 + } 314 + return retryClient.StandardClient() 315 + }