ai cooking
bright data proxy? (#428)
* bright data proxy?
* remove bd proxy from scrapers
* move tiemout out of bright data
* so much trimming
* diet continues
* shorten it up
* embed bright data cert
* ignore printf returns
---------
Co-authored-by: paul miller <paul.miller>
authored by
Paul Miller
paul miller
and committed by
GitHub
bf1661c9
9f7411af
+343
-98
+16
-19
cmd/albertsonsquery/main.go
+16
-19
cmd/albertsonsquery/main.go
···
5
5
"errors"
6
6
"flag"
7
7
"fmt"
8
-
"io"
9
-
"net/http"
10
8
"os"
11
9
"time"
12
10
13
11
"careme/internal/albertsons/query"
12
+
"careme/internal/brightdata"
14
13
)
15
14
16
15
func main() {
17
-
if err := run(context.Background(), os.Stdout, os.Args[1:]); err != nil {
16
+
if err := run(context.Background(), os.Args[1:]); err != nil {
18
17
fmt.Fprintf(os.Stderr, "error: %v\n", err)
19
18
os.Exit(1)
20
19
}
21
20
}
22
21
23
-
func run(ctx context.Context, stdout io.Writer, args []string) error {
24
-
return runWithHTTPClient(ctx, stdout, args, nil)
25
-
}
26
-
27
-
// exists just for UT
28
-
func runWithHTTPClient(ctx context.Context, stdout io.Writer, args []string, httpClient *http.Client) error {
29
-
fs := flag.NewFlagSet("albertsonsquery", flag.ContinueOnError)
30
-
fs.SetOutput(io.Discard)
22
+
func run(ctx context.Context, args []string) error {
23
+
fs := flag.NewFlagSet("albertsonsquery", flag.ExitOnError)
31
24
32
25
var (
33
26
baseURL string
···
40
33
timeoutSec int
41
34
)
42
35
36
+
// get from store_id?
43
37
fs.StringVar(&baseURL, "base-url", query.DefaultSearchBaseURL, "Albertsons-family search base URL")
44
38
fs.StringVar(&storeID, "store-id", "806", "store id to search against")
45
39
fs.StringVar(&subscriptionKey, "subscription-key", envOrDefault("ALBERTSONS_SEARCH_SUBSCRIPTION_KEY", ""), "Albertsons pathway subscription key")
···
58
52
if subscriptionKey == "" {
59
53
return errors.New("subscription-key is required")
60
54
}
61
-
62
-
// todo proxy through bright data ? timeout with context instead of http client?
63
-
if httpClient == nil {
64
-
httpClient = &http.Client{Timeout: time.Duration(timeoutSec) * time.Second}
55
+
if timeoutSec > 0 {
56
+
var cancel context.CancelFunc
57
+
ctx, cancel = context.WithTimeout(ctx, time.Duration(timeoutSec)*time.Second)
58
+
defer cancel()
59
+
}
60
+
httpClient, err := brightdata.NewProxyAwareHTTPClient(brightdata.LoadConfig())
61
+
if err != nil {
62
+
return fmt.Errorf("create HTTP client: %w", err)
65
63
}
66
64
client, err := query.NewSearchClient(query.SearchClientConfig{
67
65
BaseURL: baseURL,
···
83
81
}
84
82
85
83
for i, doc := range payload.Response.Docs {
86
-
_, _ = fmt.Fprintf(stdout, "%d: %s (price: %.2f)\n", i+1, doc.Name, doc.Price)
84
+
fmt.Printf("%d: %s (price: %.2f)\n", i+1, doc.Name, doc.Price)
87
85
}
88
-
_, err = fmt.Fprintf(stdout, "total products: %d\n", len(payload.Response.Docs))
89
-
90
-
return err
86
+
fmt.Printf("total products: %d\n", len(payload.Response.Docs))
87
+
return nil
91
88
}
92
89
93
90
func envOrDefault(key, fallback string) string {
-60
cmd/albertsonsquery/main_test.go
-60
cmd/albertsonsquery/main_test.go
···
1
-
package main
2
-
3
-
import (
4
-
"context"
5
-
"io"
6
-
"net/http"
7
-
"strings"
8
-
"testing"
9
-
)
10
-
11
-
func TestRunOutputsReturnedDocCount(t *testing.T) {
12
-
t.Parallel()
13
-
14
-
httpClient := &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
15
-
if r.URL.Path != "/abs/pub/xapi/wcax/pathway/search" {
16
-
t.Fatalf("unexpected path: %s", r.URL.Path)
17
-
}
18
-
if got := r.URL.Query().Get("storeid"); got != "806" {
19
-
t.Fatalf("unexpected storeid: %q", got)
20
-
}
21
-
if got := r.Header.Get("Ocp-Apim-Subscription-Key"); got != "test-key" {
22
-
t.Fatalf("unexpected subscription key: %q", got)
23
-
}
24
-
25
-
return &http.Response{
26
-
StatusCode: http.StatusOK,
27
-
Header: http.Header{
28
-
"Content-Type": []string{"application/json"},
29
-
},
30
-
Body: io.NopCloser(strings.NewReader(`{
31
-
"response":{"numFound":3,"disableTracking":false,"start":0,"miscInfo":{"attributionToken":"","query":"","sort":"","filter":"","nextPageToken":""},"isExactMatch":true,"docs":[{"id":"1","name":"Apples","price":1.99},{"id":"2","name":"Bananas","price":2.49},{"id":"3","name":"Carrots","price":3.99}]},
32
-
"offersData":{"departments":{},"upcs":{}},
33
-
"facet":{"ranges":[],"fields":[],"dynamic_facets":[]},
34
-
"appCode":"ok",
35
-
"appMsg":"ok",
36
-
"dynamic_filters":{}
37
-
}`)),
38
-
}, nil
39
-
})}
40
-
41
-
var stdout strings.Builder
42
-
err := runWithHTTPClient(context.Background(), &stdout, []string{
43
-
"-base-url", "https://www.acmemarkets.com",
44
-
"-store-id", "806",
45
-
"-subscription-key", "test-key",
46
-
}, httpClient)
47
-
if err != nil {
48
-
t.Fatalf("run returned error: %v", err)
49
-
}
50
-
51
-
if got := stdout.String(); got != "1: Apples (price: 1.99)\n2: Bananas (price: 2.49)\n3: Carrots (price: 3.99)\ntotal products: 3\n" {
52
-
t.Fatalf("unexpected stdout: %q", got)
53
-
}
54
-
}
55
-
56
-
type roundTripFunc func(*http.Request) (*http.Response, error)
57
-
58
-
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
59
-
return f(r)
60
-
}
+3
-1
internal/albertsons/staples.go
+3
-1
internal/albertsons/staples.go
···
5
5
"encoding/json"
6
6
"fmt"
7
7
"log/slog"
8
+
"net/http"
8
9
"strings"
9
10
10
11
"careme/internal/albertsons/query"
···
34
35
return identityProvider{}
35
36
}
36
37
37
-
func NewStaplesProvider(cfg config.AlbertsonsConfig) StaplesProvider {
38
+
func NewStaplesProvider(cfg config.AlbertsonsConfig, httpClient *http.Client) StaplesProvider {
38
39
return newStaplesProviderWithFactory(func(baseURL string) (searchClient, error) {
39
40
querycfg := query.SearchClientConfig{
40
41
SubscriptionKey: cfg.SearchSubscriptionKey,
41
42
Reese84: cfg.SearchReese84,
42
43
BaseURL: baseURL,
44
+
HTTPClient: httpClient,
43
45
}
44
46
return query.NewSearchClient(querycfg)
45
47
})
+57
internal/albertsons/staples_test.go
+57
internal/albertsons/staples_test.go
···
3
3
import (
4
4
"context"
5
5
"encoding/json"
6
+
"io"
7
+
"net/http"
6
8
"slices"
9
+
"strings"
7
10
"sync"
8
11
"testing"
9
12
10
13
"careme/internal/albertsons/query"
14
+
"careme/internal/config"
11
15
)
12
16
13
17
func TestIdentityProviderSignature_UsesStapleCategories(t *testing.T) {
···
165
169
t.Fatalf("unexpected description: %+v", got[0].Description)
166
170
}
167
171
}
172
+
173
+
func TestNewStaplesProvider_UsesInjectedHTTPClient(t *testing.T) {
174
+
t.Parallel()
175
+
176
+
var sawRequest bool
177
+
httpClient := &http.Client{
178
+
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
179
+
sawRequest = true
180
+
if got, want := r.URL.Host, "www.acmemarkets.com"; got != want {
181
+
t.Fatalf("unexpected host: got %q want %q", got, want)
182
+
}
183
+
if got, want := r.Header.Get("Ocp-Apim-Subscription-Key"), "test-sub-key"; got != want {
184
+
t.Fatalf("unexpected subscription key: got %q want %q", got, want)
185
+
}
186
+
return &http.Response{
187
+
StatusCode: http.StatusOK,
188
+
Header: http.Header{
189
+
"Content-Type": []string{"application/json"},
190
+
},
191
+
Body: io.NopCloser(strings.NewReader(`{
192
+
"response":{"docs":[
193
+
{"id":"wine-1","name":"Pinot Noir","price":10.99},
194
+
{"id":"wine-2","name":"Rose","price":8.99}
195
+
]}
196
+
}`)),
197
+
}, nil
198
+
}),
199
+
}
200
+
201
+
provider := NewStaplesProvider(config.AlbertsonsConfig{
202
+
SearchSubscriptionKey: "test-sub-key",
203
+
}, httpClient)
204
+
205
+
got, err := provider.GetIngredients(t.Context(), "acmemarkets_806", "pinot", 1)
206
+
if err != nil {
207
+
t.Fatalf("GetIngredients returned error: %v", err)
208
+
}
209
+
if !sawRequest {
210
+
t.Fatal("expected injected HTTP client to be used")
211
+
}
212
+
if len(got) != 1 {
213
+
t.Fatalf("expected 1 ingredient after skip, got %d", len(got))
214
+
}
215
+
if got[0].Description == nil || *got[0].Description != "Rose" {
216
+
t.Fatalf("unexpected description: %+v", got[0].Description)
217
+
}
218
+
}
219
+
220
+
type roundTripFunc func(*http.Request) (*http.Response, error)
221
+
222
+
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
223
+
return f(r)
224
+
}
+23
internal/brightdata/brightdata.crt
+23
internal/brightdata/brightdata.crt
···
1
+
-----BEGIN CERTIFICATE-----
2
+
MIID1DCCArygAwIBAgIUeAKxfu5Akc2vs63HjJCeOqPVpC8wDQYJKoZIhvcNAQEL
3
+
BQAwVDELMAkGA1UEBhMCSUwxCzAJBgNVBAgMAklMMRQwEgYDVQQKDAtCcmlnaHQg
4
+
RGF0YTEiMCAGA1UEAwwZQnJpZ2h0IERhdGEgUHJveHkgUm9vdCBDQTAeFw0yNDA5
5
+
MTYxNTE3NTVaFw0zNDA5MTQxNTE3NTVaMFQxCzAJBgNVBAYTAklMMQswCQYDVQQI
6
+
DAJJTDEUMBIGA1UECgwLQnJpZ2h0IERhdGExIjAgBgNVBAMMGUJyaWdodCBEYXRh
7
+
IFByb3h5IFJvb3QgQ0EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC/
8
+
ECdI43fgos9VnyJecnW2KsnClOMuzACmxSdCYzfl/2RgS/d4HludE8hXjkbT8ive
9
+
Nb5+Wa6fAB3cOmDPXTdYkHqGbWGGap3YQhsyC5DqxZwKPgWBshIP4kai8CL8UzRh
10
+
rDz75KW5G12amr7ozQxYnrkyr7DIz3Reac/9ShuZVr8iRdQI2yJCeP1uw0VYt77m
11
+
3dvw2XVar1m79TawBXws1twORbu5I+qA31YSDEs4CAfTcK+405TcZWLngjUb2Rtj
12
+
EHF/qHcQuXrS5rpAskXWj0n45UFqPg5stJRnigT/DzV4BhOkvSNKh4TAg1202TIT
13
+
/OCDF9heAks6jVjAHx1DAgMBAAGjgZ0wgZowHQYDVR0OBBYEFOmeH8XNfK2OcWef
14
+
fztuOmD+XHbjMB8GA1UdIwQYMBaAFOmeH8XNfK2OcWeffztuOmD+XHbjMA8GA1Ud
15
+
EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMDcGA1UdHwQwMC4wLKAqoCiGJmh0
16
+
dHA6Ly9jcmwuYnJpZ2h0ZGF0YS5jb20vcHJveHlfY2EuY3JsMA0GCSqGSIb3DQEB
17
+
CwUAA4IBAQAqmunmZM+/9ClQjzXaHmBu3I/eLqqWOssnJYA7tfK+EVXAgUYULJ5u
18
+
BO3FIWdhSX1HvmOz7nodNru7z2JDngAHVmCVpLJ82a8s8RK1OPtDZE+j50UVNk/G
19
+
McXVFFrL1AyAL6TgANTdGYtNNHHIb2HcOnYNwDqIvqJZn43FF9UWSY1sh3vOMbmI
20
+
geXK0RG6XMVKu7/abhWFBZBTFNsbuu8Qf4C+SGwvO6qHVmqV1iLgGlgnYNhksOMq
21
+
M3lc6F9H0Qpe91+5mMvfienrWPYOOvFFKPMCX714cZGUUlKpGoO5FibqVCmYf8kY
22
+
7TxA62alCsJJv4K1np6UXdPmCHQVV9yI
23
+
-----END CERTIFICATE-----
+83
internal/brightdata/proxy.go
+83
internal/brightdata/proxy.go
···
1
+
package brightdata
2
+
3
+
import (
4
+
"crypto/tls"
5
+
"crypto/x509"
6
+
_ "embed"
7
+
"fmt"
8
+
"log/slog"
9
+
"net"
10
+
"net/http"
11
+
"net/url"
12
+
"os"
13
+
)
14
+
15
+
//go:embed brightdata.crt
16
+
var brightDataRootCA []byte
17
+
18
+
type ProxyConfig struct {
19
+
Host string `json:"host"`
20
+
Port string `json:"port"`
21
+
Username string `json:"username"`
22
+
Password string `json:"password"`
23
+
}
24
+
25
+
func LoadConfig() ProxyConfig {
26
+
return ProxyConfig{
27
+
Host: os.Getenv("BRIGHTDATA_PROXY_HOST"),
28
+
Port: os.Getenv("BRIGHTDATA_PROXY_PORT"),
29
+
Username: os.Getenv("BRIGHTDATA_PROXY_USERNAME"),
30
+
Password: os.Getenv("BRIGHTDATA_PROXY_PASSWORD"),
31
+
}
32
+
}
33
+
34
+
func (c ProxyConfig) Enabled() bool {
35
+
return c.Host != "" && c.Port != "" && c.Username != "" && c.Password != ""
36
+
}
37
+
38
+
func (c ProxyConfig) proxyURL() *url.URL {
39
+
return &url.URL{
40
+
Scheme: "http",
41
+
User: url.UserPassword(c.Username, c.Password),
42
+
Host: net.JoinHostPort(c.Host, c.Port),
43
+
}
44
+
}
45
+
46
+
func NewProxyAwareHTTPClient(cfg ProxyConfig) (*http.Client, error) {
47
+
client := &http.Client{}
48
+
if !cfg.Enabled() {
49
+
return client, nil
50
+
}
51
+
52
+
rootCAs, err := proxyRootCAs()
53
+
if err != nil {
54
+
return nil, err
55
+
}
56
+
57
+
slog.Info(
58
+
"Configuring HTTP client to use BrightData proxy",
59
+
"host", cfg.Host,
60
+
"port", cfg.Port,
61
+
"username", cfg.Username,
62
+
)
63
+
64
+
transport := http.DefaultTransport.(*http.Transport).Clone()
65
+
transport.Proxy = http.ProxyURL(cfg.proxyURL())
66
+
transport.TLSClientConfig = &tls.Config{RootCAs: rootCAs}
67
+
client.Transport = transport
68
+
return client, nil
69
+
}
70
+
71
+
func proxyRootCAs() (*x509.CertPool, error) {
72
+
pool, err := x509.SystemCertPool()
73
+
if err != nil {
74
+
return nil, fmt.Errorf("load system cert pool: %w", err)
75
+
}
76
+
if pool == nil {
77
+
pool = x509.NewCertPool()
78
+
}
79
+
if ok := pool.AppendCertsFromPEM(brightDataRootCA); !ok {
80
+
return nil, fmt.Errorf("append embedded BrightData root CA")
81
+
}
82
+
return pool, nil
83
+
}
+97
internal/brightdata/proxy_test.go
+97
internal/brightdata/proxy_test.go
···
1
+
package brightdata
2
+
3
+
import (
4
+
"net/http"
5
+
"testing"
6
+
)
7
+
8
+
func TestProxyConfigValidate_AllowsDisabled(t *testing.T) {
9
+
t.Parallel()
10
+
11
+
if (ProxyConfig{}).Enabled() {
12
+
t.Fatalf("Empty config should not be enabled")
13
+
}
14
+
}
15
+
16
+
func TestProxyConfigValidate_RejectsPartialConfig(t *testing.T) {
17
+
t.Parallel()
18
+
19
+
enabled := (ProxyConfig{
20
+
Host: "brd.superproxy.io",
21
+
Port: "33335",
22
+
}).Enabled()
23
+
if enabled {
24
+
t.Fatal("expected diabled when only host and port provided")
25
+
}
26
+
}
27
+
28
+
func TestProxyConfigProxyURL_BuildsProxyURL(t *testing.T) {
29
+
t.Parallel()
30
+
31
+
proxyURL := (ProxyConfig{
32
+
Host: "brd.superproxy.io",
33
+
Port: "33335",
34
+
Username: "user-name",
35
+
Password: "secret-pass",
36
+
}).proxyURL()
37
+
38
+
if got, want := proxyURL.String(), "http://user-name:secret-pass@brd.superproxy.io:33335"; got != want {
39
+
t.Fatalf("unexpected proxy URL: got %q want %q", got, want)
40
+
}
41
+
}
42
+
43
+
func TestNewProxyAwareHTTPClient_UsesConfiguredProxy(t *testing.T) {
44
+
t.Parallel()
45
+
46
+
client, err := NewProxyAwareHTTPClient(ProxyConfig{
47
+
Host: "brd.superproxy.io",
48
+
Port: "33335",
49
+
Username: "user-name",
50
+
Password: "secret-pass",
51
+
})
52
+
if err != nil {
53
+
t.Fatalf("NewProxyAwareHTTPClient() error = %v", err)
54
+
}
55
+
56
+
if client.Timeout != 0 {
57
+
t.Fatalf("expected no client timeout, got %s", client.Timeout)
58
+
}
59
+
60
+
transport, ok := client.Transport.(*http.Transport)
61
+
if !ok {
62
+
t.Fatalf("expected *http.Transport, got %T", client.Transport)
63
+
}
64
+
65
+
req, err := http.NewRequest(http.MethodGet, "https://www.example.com/products", nil)
66
+
if err != nil {
67
+
t.Fatalf("NewRequest() error = %v", err)
68
+
}
69
+
proxyURL, err := transport.Proxy(req)
70
+
if err != nil {
71
+
t.Fatalf("transport.Proxy() error = %v", err)
72
+
}
73
+
if proxyURL == nil {
74
+
t.Fatal("expected proxy URL")
75
+
}
76
+
if got, want := proxyURL.String(), "http://user-name:secret-pass@brd.superproxy.io:33335"; got != want {
77
+
t.Fatalf("unexpected proxy URL: got %q want %q", got, want)
78
+
}
79
+
if transport.TLSClientConfig == nil || transport.TLSClientConfig.RootCAs == nil {
80
+
t.Fatal("expected embedded BrightData CA pool to be configured")
81
+
}
82
+
}
83
+
84
+
func TestNewProxyAwareHTTPClient_DisabledLeavesDefaultTransport(t *testing.T) {
85
+
t.Parallel()
86
+
87
+
client, err := NewProxyAwareHTTPClient(ProxyConfig{})
88
+
if err != nil {
89
+
t.Fatalf("NewProxyAwareHTTPClient() error = %v", err)
90
+
}
91
+
if client.Timeout != 0 {
92
+
t.Fatalf("expected no client timeout, got %s", client.Timeout)
93
+
}
94
+
if client.Transport != nil {
95
+
t.Fatalf("expected nil transport when proxy disabled, got %T", client.Transport)
96
+
}
97
+
}
+17
-13
internal/config/config.go
+17
-13
internal/config/config.go
···
6
6
"net/url"
7
7
"os"
8
8
"strings"
9
+
10
+
"careme/internal/brightdata"
9
11
)
10
12
11
13
const additionalStoresEnableEnv = "EXTRA_STORES_ENABLE"
···
15
17
)
16
18
17
19
type Config struct {
18
-
AI AIConfig `json:"ai"`
19
-
Kroger KrogerConfig `json:"kroger"`
20
-
Walmart WalmartConfig `json:"walmart"`
21
-
Aldi AldiConfig `json:"aldi"`
22
-
WholeFoods WholeFoodsConfig `json:"wholefoods"`
23
-
Albertsons AlbertsonsConfig `json:"albertsons"`
24
-
Publix PublixConfig `json:"publix"`
25
-
HEB HEBConfig `json:"heb"`
26
-
Wegmans WegmansConfig `json:"wegmans"`
27
-
Mocks MockConfig `json:"mocks"`
28
-
Clerk ClerkConfig `json:"clerk"`
29
-
Admin AdminConfig `json:"admin"`
30
-
PublicOrigin string `json:"public_origin"`
20
+
AI AIConfig `json:"ai"`
21
+
Kroger KrogerConfig `json:"kroger"`
22
+
Walmart WalmartConfig `json:"walmart"`
23
+
Aldi AldiConfig `json:"aldi"`
24
+
WholeFoods WholeFoodsConfig `json:"wholefoods"`
25
+
Albertsons AlbertsonsConfig `json:"albertsons"`
26
+
Publix PublixConfig `json:"publix"`
27
+
HEB HEBConfig `json:"heb"`
28
+
Wegmans WegmansConfig `json:"wegmans"`
29
+
BrightDataProxy brightdata.ProxyConfig `json:"brightdata_proxy"`
30
+
Mocks MockConfig `json:"mocks"`
31
+
Clerk ClerkConfig `json:"clerk"`
32
+
Admin AdminConfig `json:"admin"`
33
+
PublicOrigin string `json:"public_origin"`
31
34
}
32
35
33
36
type AIConfig struct {
···
183
186
Wegmans: WegmansConfig{
184
187
Enable: envEnabled("WEGMANS_ENABLE"),
185
188
},
189
+
BrightDataProxy: brightdata.LoadConfig(),
186
190
Walmart: WalmartConfig{
187
191
ConsumerID: os.Getenv("WALMART_CONSUMER_ID"),
188
192
KeyVersion: os.Getenv("WALMART_KEY_VERSION"),
+31
internal/config/config_test.go
+31
internal/config/config_test.go
···
93
93
}
94
94
}
95
95
96
+
func TestLoadReadsBrightDataProxyConfig(t *testing.T) {
97
+
resetStoreEnvs(t)
98
+
t.Setenv("ENABLE_MOCKS", "1")
99
+
t.Setenv("BRIGHTDATA_PROXY_HOST", "brd.superproxy.io")
100
+
t.Setenv("BRIGHTDATA_PROXY_PORT", "33335")
101
+
t.Setenv("BRIGHTDATA_PROXY_USERNAME", "brd-customer-test-zone-residential")
102
+
t.Setenv("BRIGHTDATA_PROXY_PASSWORD", "secret")
103
+
104
+
cfg, err := Load()
105
+
if err != nil {
106
+
t.Fatalf("Load() error = %v", err)
107
+
}
108
+
109
+
if got, want := cfg.BrightDataProxy.Host, "brd.superproxy.io"; got != want {
110
+
t.Fatalf("expected bright data host %q, got %q", want, got)
111
+
}
112
+
if got, want := cfg.BrightDataProxy.Port, "33335"; got != want {
113
+
t.Fatalf("expected bright data port %q, got %q", want, got)
114
+
}
115
+
if got, want := cfg.BrightDataProxy.Username, "brd-customer-test-zone-residential"; got != want {
116
+
t.Fatalf("expected bright data username %q, got %q", want, got)
117
+
}
118
+
if got, want := cfg.BrightDataProxy.Password, "secret"; got != want {
119
+
t.Fatalf("expected bright data password %q, got %q", want, got)
120
+
}
121
+
}
122
+
96
123
func TestResolvedPublicOriginDefaultsToLocalhostOutsideProd(t *testing.T) {
97
124
cfg := &Config{}
98
125
if got, want := cfg.ResolvedPublicOrigin(), "http://localhost:8080"; got != want {
···
140
167
"ALBERTSONS_ENABLE",
141
168
"ALBERTSONS_SEARCH_SUBSCRIPTION_KEY",
142
169
"ALBERTSONS_SEARCH_REESE84",
170
+
"BRIGHTDATA_PROXY_HOST",
171
+
"BRIGHTDATA_PROXY_PORT",
172
+
"BRIGHTDATA_PROXY_USERNAME",
173
+
"BRIGHTDATA_PROXY_PASSWORD",
143
174
"PUBLIX_ENABLE",
144
175
"HEB_ENABLE",
145
176
} {
+16
-5
internal/recipes/staples.go
+16
-5
internal/recipes/staples.go
···
6
6
"testing"
7
7
8
8
"careme/internal/albertsons"
9
+
"careme/internal/brightdata"
9
10
"careme/internal/config"
10
11
"careme/internal/kroger"
11
12
"careme/internal/walmart"
···
38
39
if err != nil {
39
40
return nil, err
40
41
}
42
+
backends, err := defaultStaplesBackends(cfg, kclient)
43
+
if err != nil {
44
+
return nil, err
45
+
}
46
+
41
47
return routingStaplesProvider{
42
-
backends: defaultStaplesBackends(cfg, kclient),
48
+
backends: backends,
43
49
}, nil
44
50
}
45
51
···
82
88
return nil, fmt.Errorf("staples provider does not support location %q", locationID)
83
89
}
84
90
85
-
func defaultStaplesBackends(cfg *config.Config, krogerClient kroger.ClientWithResponsesInterface) []backendStaplesProvider {
91
+
func defaultStaplesBackends(cfg *config.Config, krogerClient kroger.ClientWithResponsesInterface) ([]backendStaplesProvider, error) {
92
+
httpClient, err := brightdata.NewProxyAwareHTTPClient(cfg.BrightDataProxy)
93
+
if err != nil {
94
+
return nil, fmt.Errorf("create bright data proxy-aware client: %w", err)
95
+
}
96
+
86
97
return []backendStaplesProvider{
87
98
kroger.NewStaplesProvider(krogerClient),
88
99
// actowiz.NewStaplesProvider(),
89
100
walmart.NewStaplesProvider(),
90
-
wholefoods.NewStaplesProvider(wholefoods.NewClient(nil)),
91
-
albertsons.NewStaplesProvider(cfg.Albertsons),
92
-
}
101
+
wholefoods.NewStaplesProvider(wholefoods.NewClient(httpClient)),
102
+
albertsons.NewStaplesProvider(cfg.Albertsons, httpClient),
103
+
}, nil
93
104
}
94
105
95
106
func defaultIdentityProviders() []identityProvider {