ai cooking
Resse84 (#434)
* what does reese cachign look like
* try and generate reese tokens
* okay got broser to get me the cookie
* remove unlocker
* remove old auth
* a little cleaner
* generize to cookie
* remove fallback
* don't normalize
* remove struct that doesn't do much
* comments
* good bye testify
---------
Co-authored-by: paul miller <paul.miller>
authored by
Paul Miller
paul miller
and committed by
GitHub
bbc51b83
75ff6cda
+1043
-72
+3
.github/workflows/go.yml
+3
.github/workflows/go.yml
+4
README.md
+4
README.md
···
17
17
- `GOOGLE_TAG_ID` - Google Ads/gtag ID for web analytics (optional)
18
18
- `GOOGLE_CONVERSION_LABEL` - Google Ads conversion label used on `/auth/establish?signup=true` (optional)
19
19
- `SENDGRID_API_KEY` - To allow sending weekly recipe lists via email
20
+
- `ALBERTSONS_SEARCH_SUBSCRIPTION_KEY` - Albertsons-family pathway search subscription key
21
+
- `ALBERTSONS_SEARCH_REESE84` - fallback Albertsons-family `reese84` cookie when cache is empty or stale
22
+
- `BRIGHTDATA_BROWSER_WS_ENDPOINT` - Bright Data Browser API websocket endpoint for `cmd/albertsonsreese84`; may include embedded credentials
23
+
- `AZURE_STORAGE_ACCOUNT_NAME` and `AZURE_STORAGE_PRIMARY_ACCOUNT_KEY` - enable Azure Blob-backed cache storage
20
24
21
25
if you're
22
26
- `ENABLE_MOCKS` - For testing if you have none of the above
+4
-2
cmd/albertsonsquery/main.go
+4
-2
cmd/albertsonsquery/main.go
···
64
64
client, err := query.NewSearchClient(query.SearchClientConfig{
65
65
BaseURL: baseURL,
66
66
SubscriptionKey: subscriptionKey,
67
-
Reese84: reese84,
68
-
HTTPClient: httpClient,
67
+
Reese84Provider: func(context.Context) (string, error) {
68
+
return reese84, nil
69
+
}, // use the alberttson cache?
70
+
HTTPClient: httpClient,
69
71
})
70
72
if err != nil {
71
73
return fmt.Errorf("create search client: %w", err)
+100
cmd/albertsonsreese84/main.go
+100
cmd/albertsonsreese84/main.go
···
1
+
package main
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"flag"
7
+
"fmt"
8
+
"io"
9
+
"log"
10
+
"os"
11
+
"strings"
12
+
"time"
13
+
14
+
"careme/internal/albertsons"
15
+
"careme/internal/brightdata"
16
+
"careme/internal/cache"
17
+
"careme/internal/logsetup"
18
+
)
19
+
20
+
const (
21
+
// should we rotate between chains?
22
+
defaultTargetURL = "https://www.acmemarkets.com/aisle-vs/meat-seafood/seafood-favorites.html"
23
+
defaultCookieName = "reese84"
24
+
brightDataBrowserWSEnv = "BRIGHTDATA_BROWSER_WS_ENDPOINT"
25
+
)
26
+
27
+
func main() {
28
+
ctx := context.Background()
29
+
closeLogger, err := logsetup.Configure(ctx)
30
+
if err != nil {
31
+
log.Fatalf("failed to configure logging: %v", err)
32
+
}
33
+
defer closeLogger()
34
+
35
+
if err := runWithDeps(ctx, os.Args[1:]); err != nil {
36
+
fmt.Fprintf(os.Stderr, "error: %v\n", err)
37
+
os.Exit(1)
38
+
}
39
+
}
40
+
41
+
func runWithDeps(ctx context.Context, args []string) error {
42
+
fs := flag.NewFlagSet("albertsonsreese84", flag.ContinueOnError)
43
+
fs.SetOutput(io.Discard)
44
+
45
+
var (
46
+
targetURL string
47
+
cookieName string
48
+
wsEndpoint string
49
+
waitMS int
50
+
timeoutSec int
51
+
)
52
+
53
+
fs.StringVar(&targetURL, "url", defaultTargetURL, "page to navigate before reading cookies")
54
+
fs.StringVar(&cookieName, "cookie-name", defaultCookieName, "cookie name to store")
55
+
fs.StringVar(&wsEndpoint, "ws-endpoint", strings.TrimSpace(os.Getenv(brightDataBrowserWSEnv)), "Bright Data Browser API websocket endpoint including credentials")
56
+
fs.IntVar(&waitMS, "wait-ms", int((5*time.Second)/time.Millisecond), "wait after initial navigation before reading cookies")
57
+
fs.IntVar(&timeoutSec, "timeout", 120, "overall timeout in seconds")
58
+
59
+
if err := fs.Parse(args); err != nil {
60
+
return err
61
+
}
62
+
if strings.TrimSpace(cookieName) == "" {
63
+
return errors.New("cookie-name is required")
64
+
}
65
+
wsEndpoint = strings.TrimSpace(wsEndpoint)
66
+
if wsEndpoint == "" {
67
+
return fmt.Errorf("%s is required", brightDataBrowserWSEnv)
68
+
}
69
+
70
+
fetchCtx, cancel := context.WithTimeout(ctx, time.Duration(timeoutSec)*time.Second)
71
+
defer cancel()
72
+
73
+
browser, err := brightdata.NewBrowserClient(brightdata.BrowserClientConfig{
74
+
WSEndpoint: wsEndpoint,
75
+
})
76
+
if err != nil {
77
+
return fmt.Errorf("create Bright Data browser client: %w", err)
78
+
}
79
+
80
+
record, err := albertsons.FetchCookie(fetchCtx, browser, albertsons.CookieParams{
81
+
TargetURL: targetURL,
82
+
CookieName: cookieName,
83
+
WaitAfterNavigation: time.Duration(waitMS) * time.Millisecond,
84
+
})
85
+
if err != nil {
86
+
return err
87
+
}
88
+
89
+
cacheStore, err := cache.EnsureCache(albertsons.Container)
90
+
if err != nil {
91
+
return fmt.Errorf("create albertsons cache: %w", err)
92
+
}
93
+
94
+
if err := albertsons.SaveReese84Record(fetchCtx, cacheStore, record); err != nil {
95
+
return fmt.Errorf("cache reese84 cookie: %w", err)
96
+
}
97
+
98
+
fmt.Printf("cached %s at %s from %s\n", cookieName, record.FetchedAt.Format(time.RFC3339), targetURL)
99
+
return nil
100
+
}
+44
deploy/cronjob-albertsons-reese84.yaml
+44
deploy/cronjob-albertsons-reese84.yaml
···
1
+
apiVersion: batch/v1
2
+
kind: CronJob
3
+
metadata:
4
+
name: albertsons-reese84
5
+
labels:
6
+
app: albertsons-reese84
7
+
spec:
8
+
schedule: "0 */6 * * *"
9
+
concurrencyPolicy: Forbid
10
+
successfulJobsHistoryLimit: 2
11
+
failedJobsHistoryLimit: 3
12
+
jobTemplate:
13
+
spec:
14
+
backoffLimit: 1
15
+
template:
16
+
metadata:
17
+
labels:
18
+
app: albertsons-reese84
19
+
job: albertsons-reese84
20
+
spec:
21
+
restartPolicy: Never
22
+
securityContext:
23
+
runAsNonRoot: true
24
+
runAsUser: 65532
25
+
runAsGroup: 65532
26
+
containers:
27
+
- name: albertsons-reese84
28
+
image: ghcr.io/paulgmiller/careme-albertsonsreese84:${IMAGE_TAG}
29
+
imagePullPolicy: IfNotPresent
30
+
envFrom:
31
+
- secretRef:
32
+
name: storage
33
+
- secretRef:
34
+
name: brightdata
35
+
env:
36
+
- name: APPLICATIONINSIGHTS_CONNECTION_STRING
37
+
value: "InstrumentationKey=a532fcc7-5098-4f44-8dde-ff2f32d6a59b;IngestionEndpoint=https://westus3-1.in.applicationinsights.azure.com/;LiveEndpoint=https://westus3.livediagnostics.monitor.azure.com/;ApplicationId=fdc94780-6135-4a29-980e-ab114a402e58"
38
+
resources:
39
+
requests:
40
+
cpu: 50m
41
+
memory: 64Mi
42
+
limits:
43
+
cpu: 500m
44
+
memory: 256Mi
+3
docs/cache-layout.md
+3
docs/cache-layout.md
···
42
42
| `albertsons/stores/` | JSON `albertsons.StoreSummary` keyed by prefixed Albertsons-family location ID | `cmd/albertsons` and `internal/albertsons` cache helpers | `internal/albertsons` location backend |
43
43
| `albertsons/store_locations.json` | JSON `[]storeindex.Entry` spatial index for Albertsons-family stores (`id`, `lat`, `lon`) | `cmd/albertsons` rebuilds after sync | `internal/albertsons` location backend |
44
44
| `albertsons/store_url_map.json` | JSON object mapping store URL to prefixed Albertsons-family location ID | `cmd/albertsons` and `internal/albertsons` cache helpers | `cmd/albertsons` incremental sync |
45
+
| `albertsons/reese84/latest.json` | JSON `albertsons.Reese84Record` containing the freshest ACME/Albertsons-family `reese84` cookie plus metadata | `cmd/albertsonsreese84` | `internal/albertsons` staples/search cookie resolver |
46
+
| `albertsons/reese84/history/` | JSON `albertsons.Reese84Record` append-only history keyed by fetch timestamp | `cmd/albertsonsreese84` | Operational debugging and manual rollback/reference |
45
47
| `aldi/store_locations.json` | JSON `[]storeindex.Entry` spatial index for ALDI stores (`id`, `lat`, `lon`) | `cmd/aldi` rebuilds after sync | `internal/aldi` location backend |
46
48
| `heb/stores/` | JSON `heb.StoreSummary` keyed by prefixed HEB location ID | `cmd/heb` and `internal/heb` cache helpers | `internal/heb` location backend |
47
49
| `heb/store_locations.json` | JSON `[]storeindex.Entry` spatial index for HEB stores (`id`, `lat`, `lon`) | `cmd/heb` rebuilds after sync | `internal/heb` location backend |
···
62
64
- Most app caches use the default cache created via `cache.MakeCache()` / `cache.EnsureCache("recipes")`.
63
65
- ALDI locations use a separate cache created via `cache.EnsureCache("aldi")`.
64
66
- Albertsons-family locations use a separate cache created via `cache.EnsureCache("albertsons")`.
67
+
- Albertsons-family `reese84` cookie refresh also uses `cache.EnsureCache("albertsons")`; the latest record is overwritten while timestamped history remains append-only.
65
68
- Wegmans locations use a separate cache created via `cache.EnsureCache("wegmans")`.
66
69
- HEB locations use a separate cache created via `cache.EnsureCache("heb")`.
67
70
- Publix uses a separate cache created via `cache.EnsureCache("publix")`; it does not share the `recipes` container/directory.
+5
-2
go.mod
+5
-2
go.mod
···
10
10
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
11
11
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
12
12
github.com/clerk/clerk-sdk-go/v2 v2.5.1
13
+
github.com/gobwas/ws v1.4.0
13
14
github.com/invopop/jsonschema v0.13.0
14
15
github.com/microsoft/ApplicationInsights-Go v0.4.4
15
16
github.com/openai/openai-go/v3 v3.29.0
···
18
19
github.com/sendgrid/sendgrid-go v3.16.1+incompatible
19
20
golang.org/x/crypto v0.49.0
20
21
golang.org/x/net v0.52.0
22
+
golang.org/x/sync v0.20.0
21
23
)
22
24
23
25
require (
···
26
28
github.com/bahlo/generic-list-go v0.2.0 // indirect
27
29
github.com/buger/jsonparser v1.1.1 // indirect
28
30
github.com/go-jose/go-jose/v3 v3.0.4 // indirect
31
+
github.com/gobwas/httphead v0.1.0 // indirect
32
+
github.com/gobwas/pool v0.2.1 // indirect
29
33
github.com/gofrs/uuid v3.3.0+incompatible // indirect
30
-
github.com/stretchr/testify v1.11.1 // indirect
31
34
github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
32
-
golang.org/x/sync v0.20.0 // indirect
35
+
github.com/woodsbury/decimal128 v1.3.0 // indirect
33
36
golang.org/x/sys v0.42.0 // indirect
34
37
)
35
38
+14
-42
go.sum
+14
-42
go.sum
···
1
1
code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ=
2
2
code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8=
3
-
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo=
4
-
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
5
3
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
6
4
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
7
-
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=
8
-
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=
9
5
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 h1:Hk5QBxZQC1jb2Fwj6mpzme37xbCDdNTxU7O9eb5+LB4=
10
-
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
11
-
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
6
+
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1/go.mod h1:IYus9qsFobWIc2YVwe/WPjcnyCkPKtnHAqUYeebc8z0=
12
7
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
13
8
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
14
9
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
15
10
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
16
-
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI=
17
-
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk=
18
11
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
19
12
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
20
-
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
21
-
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
22
13
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs=
14
+
github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk=
23
15
github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk=
24
16
github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
25
17
github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
···
31
23
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
32
24
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
33
25
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
34
-
github.com/clerk/clerk-sdk-go/v2 v2.5.0 h1:+haviGll3gfUNE1Y7JwGQa7vICz7RhA9dmyT5eET1Rc=
35
-
github.com/clerk/clerk-sdk-go/v2 v2.5.0/go.mod h1:VlJ9eDtVdZhugRPbguGJNMVwA7ToFOsXvjtkn20MKjE=
36
26
github.com/clerk/clerk-sdk-go/v2 v2.5.1 h1:RsakGNW6ie83b9KIRtKzqDXBJ//cURy9SJUbGhrsIKg=
37
27
github.com/clerk/clerk-sdk-go/v2 v2.5.1/go.mod h1:ncFmsPwmD5WpGCNW5bJve862j/HQfpkzsshXYV/quJ8=
38
28
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
···
44
34
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
45
35
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
46
36
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
47
-
github.com/getkin/kin-openapi v0.132.0 h1:3ISeLMsQzcb5v26yeJrBcdTCEQTag36ZjaGk7MIRUwk=
48
-
github.com/getkin/kin-openapi v0.132.0/go.mod h1:3OlG51PCYNsPByuiMB0t4fjnNlIDnaEDsjiKUV8nL58=
49
37
github.com/getkin/kin-openapi v0.133.0 h1:pJdmNohVIJ97r4AUFtEXRXwESr8b0bD721u/Tz6k8PQ=
50
38
github.com/getkin/kin-openapi v0.133.0/go.mod h1:boAciF6cXk5FhPqe/NQeBTeenbjqU4LhWBf09ILVvWE=
51
39
github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
···
57
45
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
58
46
github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM=
59
47
github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
48
+
github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU=
49
+
github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM=
50
+
github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
51
+
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
52
+
github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs=
53
+
github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc=
60
54
github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
61
55
github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
62
-
github.com/golang-jwt/jwt/v5 v5.2.3 h1:kkGXqQOBSDDWRhWNXTFpqGSCMyh/PLnqUvMGJPDJDs0=
63
-
github.com/golang-jwt/jwt/v5 v5.2.3/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
64
56
github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
57
+
github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
65
58
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
66
59
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
67
60
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
···
106
99
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
107
100
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
108
101
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
109
-
github.com/oapi-codegen/oapi-codegen/v2 v2.5.0 h1:iJvF8SdB/3/+eGOXEpsWkD8FQAHj6mqkb6Fnsoc8MFU=
110
-
github.com/oapi-codegen/oapi-codegen/v2 v2.5.0/go.mod h1:fwlMxUEMuQK5ih9aymrxKPQqNm2n8bdLk1ppjH+lr9w=
111
102
github.com/oapi-codegen/oapi-codegen/v2 v2.6.0 h1:4i+F2cvwBFZeplxCssNdLy3MhNzUD87mI3HnayHZkAU=
112
103
github.com/oapi-codegen/oapi-codegen/v2 v2.6.0/go.mod h1:eWHeJSohQJIINJZzzQriVynfGsnlQVh0UkN2UYYcw4Q=
113
-
github.com/oapi-codegen/runtime v1.1.2 h1:P2+CubHq8fO4Q6fV1tqDBZHCwpVpvPg7oKiYzQgXIyI=
114
-
github.com/oapi-codegen/runtime v1.1.2/go.mod h1:SK9X900oXmPWilYR5/WKPzt3Kqxn/uS/+lbpREv+eCg=
115
104
github.com/oapi-codegen/runtime v1.3.0 h1:vyK1zc0gDWWXgk2xoQa4+X4RNNc5SL2RbTpJS/4vMYA=
116
105
github.com/oapi-codegen/runtime v1.3.0/go.mod h1:kOdeacKy7t40Rclb1je37ZLFboFxh+YLy0zaPCMibPY=
117
106
github.com/oasdiff/yaml v0.0.0-20250309154309-f31be36b4037 h1:G7ERwszslrBzRxj//JalHPu/3yz+De2J+4aLtSRlHiY=
···
132
121
github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
133
122
github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
134
123
github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
135
-
github.com/openai/openai-go/v3 v3.25.0 h1:5sv75ZnT74mehdvzEZcRPEuFaX1I2RaCfLg6OYhgkfg=
136
-
github.com/openai/openai-go/v3 v3.25.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo=
137
124
github.com/openai/openai-go/v3 v3.29.0 h1:dZNJ0w7DxwpgppzKQjSKfLebW27KrtGqgSy4ipJS0U8=
138
125
github.com/openai/openai-go/v3 v3.29.0/go.mod h1:cdufnVK14cWcT9qA1rRtrXx4FTRsgbDPW7Ia7SS5cZo=
139
126
github.com/openclosed-dev/slogan v0.2.0 h1:Nh1z0IJ366ADFqu5pZY7SdMcYeONaeCx2J5Od9xHSfs=
···
146
133
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
147
134
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
148
135
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
149
-
github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI=
150
-
github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
151
136
github.com/samber/lo v1.53.0 h1:t975lj2py4kJPQ6haz1QMgtId2gtmfktACxIXArw3HM=
152
137
github.com/samber/lo v1.53.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0=
153
138
github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0=
···
179
164
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
180
165
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
181
166
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
182
-
github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU=
183
-
github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
184
167
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
168
+
github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg=
185
169
github.com/vmware-labs/yaml-jsonpath v0.3.2 h1:/5QKeCBGdsInyDCyVNLbXyilb61MXGi9NP674f9Hobk=
186
170
github.com/vmware-labs/yaml-jsonpath v0.3.2/go.mod h1:U6whw1z03QyqgWdgXxvVnQ90zN1BWz5V+51Ewf8k+rQ=
187
171
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
188
172
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
173
+
github.com/woodsbury/decimal128 v1.3.0 h1:8pffMNWIlC0O5vbyHWFZAt5yWvWcrHA+3ovIIjVWss0=
174
+
github.com/woodsbury/decimal128 v1.3.0/go.mod h1:C5UTmyTjW3JftjUFzOVhC20BEQa2a4ZKOB5I6Zjb+ds=
189
175
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
190
176
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
191
177
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
···
193
179
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
194
180
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
195
181
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
196
-
golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
197
-
golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
198
182
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
199
183
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
200
184
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
201
185
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
202
186
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
203
-
golang.org/x/mod v0.25.0 h1:n7a+ZbQKQA/Ysbyb0/6IbB1H/X41mKgbhfv7AfG/44w=
204
-
golang.org/x/mod v0.25.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
205
187
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
206
188
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
207
189
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
···
215
197
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
216
198
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
217
199
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
218
-
golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
219
-
golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
220
200
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
221
201
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
222
202
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
···
224
204
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
225
205
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
226
206
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
227
-
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
228
-
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
229
207
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
230
208
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
231
209
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
···
245
223
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
246
224
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
247
225
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
226
+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
248
227
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
249
228
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
250
-
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
251
-
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
252
229
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
253
230
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
254
231
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
···
256
233
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
257
234
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
258
235
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
259
-
golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
260
-
golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
261
236
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
237
+
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
262
238
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
263
239
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
264
240
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
···
266
242
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
267
243
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
268
244
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
269
-
golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
270
-
golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
271
245
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
272
246
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
273
247
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
···
275
249
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
276
250
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
277
251
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
278
-
golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo=
279
-
golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg=
280
252
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
281
253
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
282
254
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+1
-4
internal/albertsons/locations.go
+1
-4
internal/albertsons/locations.go
···
3
3
import (
4
4
"context"
5
5
"fmt"
6
-
"log/slog"
7
6
"strings"
8
7
9
8
"careme/internal/cache"
···
40
39
return nil, fmt.Errorf("create Albertsons list cache: %w", err)
41
40
}
42
41
43
-
slog.InfoContext(ctx, "ALBERTSONS invetory", "has", cfg.Albertsons.HasInventory())
44
-
45
-
return newLocationBackend(ctx, listCache, zipLookup, cfg.Albertsons.HasInventory())
42
+
return newLocationBackend(ctx, listCache, zipLookup, true /*hasInventory*/)
46
43
}
47
44
48
45
func newLocationBackend(ctx context.Context, c cache.ListCache, zipLookup centroidByZip, inventory bool) (*LocationBackend, error) {
+9
-4
internal/albertsons/query/client.go
+9
-4
internal/albertsons/query/client.go
···
40
40
type SearchClient struct {
41
41
baseURL string
42
42
subscriptionKey string
43
-
reese84 string
43
+
reese84Provider func(context.Context) (string, error) // interface?
44
44
httpClient *http.Client
45
45
}
46
46
47
47
type SearchClientConfig struct {
48
48
BaseURL string
49
49
SubscriptionKey string
50
-
Reese84 string
50
+
Reese84Provider func(context.Context) (string, error)
51
51
HTTPClient *http.Client
52
52
}
53
53
···
78
78
return &SearchClient{
79
79
baseURL: baseURL,
80
80
subscriptionKey: subscriptionKey,
81
-
reese84: strings.TrimSpace(cfg.Reese84),
81
+
reese84Provider: cfg.Reese84Provider,
82
82
httpClient: httpClient,
83
83
}, nil
84
84
}
···
116
116
req.Header.Set("Accept-Language", "en-US,en;q=0.9")
117
117
req.Header.Set("ocp-apim-subscription-key", c.subscriptionKey)
118
118
119
-
req.AddCookie(&http.Cookie{Name: "reese84", Value: c.reese84})
119
+
reese84, err := c.reese84Provider(ctx)
120
+
if err != nil {
121
+
return nil, fmt.Errorf("resolve reese84: %w", err)
122
+
}
123
+
124
+
req.AddCookie(&http.Cookie{Name: "reese84", Value: strings.TrimSpace(reese84)})
120
125
121
126
resp, err := c.httpClient.Do(req)
122
127
if err != nil {
+64
-1
internal/albertsons/query/client_test.go
+64
-1
internal/albertsons/query/client_test.go
···
2
2
3
3
import (
4
4
"context"
5
+
"errors"
5
6
"io"
6
7
"net/http"
7
8
"net/url"
···
25
26
client, err := NewSearchClient(SearchClientConfig{
26
27
BaseURL: "https://www.acmemarkets.com",
27
28
SubscriptionKey: "test-subscription-key",
28
-
Reese84: "reese-cookie",
29
+
Reese84Provider: func(context.Context) (string, error) { return "reese-cookie", nil },
29
30
HTTPClient: &http.Client{
30
31
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
31
32
capturedReq = r
···
94
95
var capturedReq *http.Request
95
96
client, err := NewSearchClient(SearchClientConfig{
96
97
SubscriptionKey: "test-subscription-key",
98
+
Reese84Provider: func(context.Context) (string, error) { return "test-reese84", nil },
97
99
HTTPClient: &http.Client{
98
100
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
99
101
capturedReq = r
···
114
116
115
117
if got := capturedReq.URL.Query().Get("url"); got != DefaultSearchBaseURL {
116
118
t.Fatalf("unexpected url query value: %q", got)
119
+
}
120
+
}
121
+
122
+
func TestSearchUsesReese84ProviderWhenConfigured(t *testing.T) {
123
+
t.Parallel()
124
+
125
+
var capturedReq *http.Request
126
+
client, err := NewSearchClient(SearchClientConfig{
127
+
SubscriptionKey: "test-subscription-key",
128
+
Reese84Provider: func(context.Context) (string, error) {
129
+
return "fresh-cookie", nil
130
+
},
131
+
HTTPClient: &http.Client{
132
+
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
133
+
capturedReq = r
134
+
return &http.Response{
135
+
StatusCode: http.StatusOK,
136
+
Body: io.NopCloser(strings.NewReader(`{}`)),
137
+
}, nil
138
+
}),
139
+
},
140
+
})
141
+
if err != nil {
142
+
t.Fatalf("NewSearchClient returned error: %v", err)
143
+
}
144
+
145
+
if _, err := client.Search(context.Background(), "806", Category_Vegatables, SearchOptions{}); err != nil {
146
+
t.Fatalf("Search returned error: %v", err)
147
+
}
148
+
149
+
reese84Cookie, err := capturedReq.Cookie("reese84")
150
+
if err != nil {
151
+
t.Fatalf("expected reese84 cookie: %v", err)
152
+
}
153
+
if reese84Cookie.Value != "fresh-cookie" {
154
+
t.Fatalf("unexpected reese84 cookie: %q", reese84Cookie.Value)
155
+
}
156
+
}
157
+
158
+
func TestSearchReturnsProviderError(t *testing.T) {
159
+
t.Parallel()
160
+
161
+
client, err := NewSearchClient(SearchClientConfig{
162
+
SubscriptionKey: "test-subscription-key",
163
+
Reese84Provider: func(context.Context) (string, error) {
164
+
return "", errors.New("boom")
165
+
},
166
+
HTTPClient: &http.Client{
167
+
Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) {
168
+
t.Fatalf("unexpected HTTP call")
169
+
return nil, nil
170
+
}),
171
+
},
172
+
})
173
+
if err != nil {
174
+
t.Fatalf("NewSearchClient returned error: %v", err)
175
+
}
176
+
177
+
_, err = client.Search(context.Background(), "806", Category_Vegatables, SearchOptions{})
178
+
if err == nil || !strings.Contains(err.Error(), "resolve reese84") {
179
+
t.Fatalf("unexpected error: %v", err)
117
180
}
118
181
}
119
182
+78
internal/albertsons/reese84.go
+78
internal/albertsons/reese84.go
···
1
+
package albertsons
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"errors"
7
+
"fmt"
8
+
"path"
9
+
"strings"
10
+
"time"
11
+
12
+
"careme/internal/cache"
13
+
)
14
+
15
+
const (
16
+
Reese84LatestCacheKey = "albertsons/reese84/latest.json"
17
+
Reese84HistoryPrefix = "albertsons/reese84/history/"
18
+
brightDataBrowserSource = "brightdata-browser-api"
19
+
)
20
+
21
+
type CookieRecord struct {
22
+
Cookie string `json:"cookie"`
23
+
FetchedAt time.Time `json:"fetched_at"`
24
+
SourceURL string `json:"source_url"`
25
+
Provider string `json:"provider"`
26
+
ExpiresAt *time.Time `json:"expires_at,omitempty"`
27
+
}
28
+
29
+
func SaveReese84Record(ctx context.Context, c cache.Cache, record CookieRecord) error {
30
+
if c == nil {
31
+
return errors.New("cache is required")
32
+
}
33
+
34
+
record.Cookie = strings.TrimSpace(record.Cookie)
35
+
if record.Cookie == "" {
36
+
return errors.New("cookie is required")
37
+
}
38
+
// other fields are optional for now
39
+
40
+
body, err := json.Marshal(record)
41
+
if err != nil {
42
+
return fmt.Errorf("marshal reese84 record: %w", err)
43
+
}
44
+
45
+
// want to have fall backsNewCachedReese84Source
46
+
historyKey := path.Join(Reese84HistoryPrefix, record.FetchedAt.Format(time.RFC3339Nano)+".json")
47
+
if err := c.Put(ctx, historyKey, string(body), cache.Unconditional()); err != nil {
48
+
return fmt.Errorf("write reese84 history: %w", err)
49
+
}
50
+
if err := c.Put(ctx, Reese84LatestCacheKey, string(body), cache.Unconditional()); err != nil {
51
+
return fmt.Errorf("write reese84 latest: %w", err)
52
+
}
53
+
return nil
54
+
}
55
+
56
+
func LoadLatestReese84(ctx context.Context, c cache.Cache) (*CookieRecord, error) {
57
+
if c == nil {
58
+
return nil, errors.New("cache is required")
59
+
}
60
+
61
+
reader, err := c.Get(ctx, Reese84LatestCacheKey)
62
+
if err != nil {
63
+
return nil, err
64
+
}
65
+
defer func() {
66
+
_ = reader.Close()
67
+
}()
68
+
69
+
var record CookieRecord
70
+
if err := json.NewDecoder(reader).Decode(&record); err != nil {
71
+
return nil, fmt.Errorf("decode reese84 record: %w", err)
72
+
}
73
+
record.Cookie = strings.TrimSpace(record.Cookie)
74
+
if record.Cookie == "" {
75
+
return nil, fmt.Errorf("decode reese84 record: cookie is empty")
76
+
}
77
+
return &record, nil
78
+
}
+62
internal/albertsons/reese84_refresh.go
+62
internal/albertsons/reese84_refresh.go
···
1
+
package albertsons
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"fmt"
7
+
"strings"
8
+
"time"
9
+
10
+
"careme/internal/brightdata"
11
+
)
12
+
13
+
type cookieBrowser interface {
14
+
Cookies(ctx context.Context, targetURL string, opts brightdata.BrowserOptions) ([]brightdata.BrowserCookie, error)
15
+
}
16
+
17
+
type CookieParams struct {
18
+
TargetURL string
19
+
CookieName string
20
+
WaitAfterNavigation time.Duration
21
+
}
22
+
23
+
func FetchCookie(ctx context.Context, browser cookieBrowser, params CookieParams) (CookieRecord, error) {
24
+
if browser == nil {
25
+
return CookieRecord{}, errors.New("browser is required")
26
+
}
27
+
28
+
params.TargetURL = strings.TrimSpace(params.TargetURL)
29
+
if params.TargetURL == "" {
30
+
return CookieRecord{}, errors.New("target URL is required")
31
+
}
32
+
33
+
params.CookieName = strings.TrimSpace(params.CookieName)
34
+
if params.CookieName == "" {
35
+
return CookieRecord{}, errors.New("cookie name is required")
36
+
}
37
+
38
+
cookies, err := browser.Cookies(ctx, params.TargetURL, brightdata.BrowserOptions{
39
+
WaitAfterNavigation: params.WaitAfterNavigation,
40
+
})
41
+
if err != nil {
42
+
return CookieRecord{}, fmt.Errorf("browser cookie fetch: %w", err)
43
+
}
44
+
45
+
cookie, ok := brightdata.CookieNamed(cookies, params.CookieName)
46
+
if !ok {
47
+
return CookieRecord{}, fmt.Errorf("cookie %q not found in browser session", params.CookieName)
48
+
}
49
+
50
+
record := CookieRecord{
51
+
Cookie: cookie.Value,
52
+
FetchedAt: time.Now().UTC(),
53
+
SourceURL: params.TargetURL,
54
+
Provider: brightDataBrowserSource,
55
+
}
56
+
if cookie.Expires != nil {
57
+
// seems to be a month on inspection?
58
+
expiresAt := *cookie.Expires
59
+
record.ExpiresAt = &expiresAt
60
+
}
61
+
return record, nil
62
+
}
+90
internal/albertsons/reese84_refresh_test.go
+90
internal/albertsons/reese84_refresh_test.go
···
1
+
package albertsons
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"testing"
7
+
"time"
8
+
9
+
"careme/internal/brightdata"
10
+
)
11
+
12
+
type stubReese84Browser struct {
13
+
cookies []brightdata.BrowserCookie
14
+
targetURL string
15
+
wait time.Duration
16
+
err error
17
+
}
18
+
19
+
func (s *stubReese84Browser) Cookies(_ context.Context, targetURL string, opts brightdata.BrowserOptions) ([]brightdata.BrowserCookie, error) {
20
+
s.targetURL = targetURL
21
+
s.wait = opts.WaitAfterNavigation
22
+
if s.err != nil {
23
+
return nil, s.err
24
+
}
25
+
return s.cookies, nil
26
+
}
27
+
28
+
func TestFetchReese84RecordReturnsRecord(t *testing.T) {
29
+
t.Parallel()
30
+
31
+
expiresAt := time.Date(2026, time.March, 30, 22, 0, 0, 0, time.UTC)
32
+
browser := &stubReese84Browser{
33
+
cookies: []brightdata.BrowserCookie{
34
+
{Name: "reese84", Value: "cookie-value", Expires: &expiresAt},
35
+
},
36
+
}
37
+
38
+
record, err := FetchCookie(context.Background(), browser, CookieParams{
39
+
TargetURL: "https://www.acmemarkets.com/aisle-vs/meat-seafood/seafood-favorites.html",
40
+
CookieName: "reese84",
41
+
WaitAfterNavigation: 2500 * time.Millisecond,
42
+
})
43
+
if err != nil {
44
+
t.Fatalf("FetchReese84Record returned error: %v", err)
45
+
}
46
+
47
+
if record.Cookie != "cookie-value" {
48
+
t.Fatalf("unexpected cookie: %q", record.Cookie)
49
+
}
50
+
if record.Provider != brightDataBrowserSource {
51
+
t.Fatalf("unexpected provider: %q", record.Provider)
52
+
}
53
+
if record.ExpiresAt == nil || !record.ExpiresAt.Equal(expiresAt) {
54
+
t.Fatalf("unexpected expiry: %+v", record.ExpiresAt)
55
+
}
56
+
if browser.targetURL != record.SourceURL {
57
+
t.Fatalf("unexpected target URL: %q", browser.targetURL)
58
+
}
59
+
if browser.wait != 2500*time.Millisecond {
60
+
t.Fatalf("unexpected wait: %s", browser.wait)
61
+
}
62
+
}
63
+
64
+
func TestFetchReese84RecordPropagatesBrowserError(t *testing.T) {
65
+
t.Parallel()
66
+
67
+
_, err := FetchCookie(context.Background(), &stubReese84Browser{
68
+
err: errors.New("boom"),
69
+
}, CookieParams{
70
+
TargetURL: "https://www.acmemarkets.com/aisle-vs/meat-seafood/seafood-favorites.html",
71
+
CookieName: "reese84",
72
+
})
73
+
if err == nil || err.Error() != "browser cookie fetch: boom" {
74
+
t.Fatalf("unexpected error: %v", err)
75
+
}
76
+
}
77
+
78
+
func TestFetchReese84RecordErrorsWhenCookieMissing(t *testing.T) {
79
+
t.Parallel()
80
+
81
+
_, err := FetchCookie(context.Background(), &stubReese84Browser{
82
+
cookies: []brightdata.BrowserCookie{{Name: "other", Value: "x"}},
83
+
}, CookieParams{
84
+
TargetURL: "https://www.acmemarkets.com/aisle-vs/meat-seafood/seafood-favorites.html",
85
+
CookieName: "reese84",
86
+
})
87
+
if err == nil || err.Error() != `cookie "reese84" not found in browser session` {
88
+
t.Fatalf("unexpected error: %v", err)
89
+
}
90
+
}
+44
internal/albertsons/reese84_test.go
+44
internal/albertsons/reese84_test.go
···
1
+
package albertsons
2
+
3
+
import (
4
+
"testing"
5
+
"time"
6
+
7
+
"careme/internal/cache"
8
+
)
9
+
10
+
func TestSaveReese84RecordWritesLatestAndHistory(t *testing.T) {
11
+
t.Parallel()
12
+
13
+
cacheStore := cache.NewInMemoryCache()
14
+
fetchedAt := time.Date(2026, time.March, 28, 12, 0, 0, 0, time.UTC)
15
+
16
+
err := SaveReese84Record(t.Context(), cacheStore, CookieRecord{
17
+
Cookie: "cookie-value",
18
+
FetchedAt: fetchedAt,
19
+
SourceURL: "https://www.acmemarkets.com/aisle-vs/meat-seafood/seafood-favorites.html",
20
+
Provider: brightDataBrowserSource,
21
+
})
22
+
if err != nil {
23
+
t.Fatalf("SaveReese84Record returned error: %v", err)
24
+
}
25
+
26
+
record, err := LoadLatestReese84(t.Context(), cacheStore)
27
+
if err != nil {
28
+
t.Fatalf("LoadLatestReese84 returned error: %v", err)
29
+
}
30
+
if record.Cookie != "cookie-value" {
31
+
t.Fatalf("unexpected cookie: %q", record.Cookie)
32
+
}
33
+
34
+
keys, err := cacheStore.List(t.Context(), Reese84HistoryPrefix, "")
35
+
if err != nil {
36
+
t.Fatalf("List returned error: %v", err)
37
+
}
38
+
if len(keys) != 1 {
39
+
t.Fatalf("expected 1 history entry, got %d", len(keys))
40
+
}
41
+
if got, want := keys[0], fetchedAt.Format(time.RFC3339Nano)+".json"; got != want {
42
+
t.Fatalf("unexpected history key: got %q want %q", got, want)
43
+
}
44
+
}
+15
-5
internal/albertsons/staples.go
+15
-5
internal/albertsons/staples.go
···
9
9
"strings"
10
10
11
11
"careme/internal/albertsons/query"
12
+
"careme/internal/cache"
12
13
"careme/internal/config"
13
14
"careme/internal/kroger"
14
15
"careme/internal/parallelism"
···
35
36
return identityProvider{}
36
37
}
37
38
38
-
func NewStaplesProvider(cfg config.AlbertsonsConfig, httpClient *http.Client) StaplesProvider {
39
+
func NewStaplesProvider(cfg config.AlbertsonsConfig, httpClient *http.Client) (StaplesProvider, error) {
40
+
c, err := cache.EnsureCache(Container)
41
+
if err != nil {
42
+
return StaplesProvider{}, fmt.Errorf("create albertsons cache: %w", err)
43
+
}
44
+
39
45
return newStaplesProviderWithFactory(func(baseURL string) (searchClient, error) {
40
46
querycfg := query.SearchClientConfig{
41
47
SubscriptionKey: cfg.SearchSubscriptionKey,
42
-
Reese84: cfg.SearchReese84,
43
-
BaseURL: baseURL,
44
-
HTTPClient: httpClient,
48
+
Reese84Provider: func(ctx context.Context) (string, error) {
49
+
// umm we should cache this and rotate on failure?
50
+
cookie, err := LoadLatestReese84(ctx, c)
51
+
return cookie.Cookie, err
52
+
},
53
+
BaseURL: baseURL,
54
+
HTTPClient: httpClient,
45
55
}
46
56
return query.NewSearchClient(querycfg)
47
-
})
57
+
}), nil
48
58
}
49
59
50
60
// only used for testing
+9
-4
internal/albertsons/staples_test.go
+9
-4
internal/albertsons/staples_test.go
···
11
11
"testing"
12
12
13
13
"careme/internal/albertsons/query"
14
-
"careme/internal/config"
15
14
)
16
15
17
16
func TestIdentityProviderSignature_UsesStapleCategories(t *testing.T) {
···
198
197
}),
199
198
}
200
199
201
-
provider := NewStaplesProvider(config.AlbertsonsConfig{
202
-
SearchSubscriptionKey: "test-sub-key",
203
-
}, httpClient)
200
+
provider := newStaplesProviderWithFactory(func(baseURL string) (searchClient, error) {
201
+
querycfg := query.SearchClientConfig{
202
+
SubscriptionKey: "test-sub-key",
203
+
Reese84Provider: func(_ context.Context) (string, error) { return "test-reese84", nil },
204
+
BaseURL: baseURL,
205
+
HTTPClient: httpClient,
206
+
}
207
+
return query.NewSearchClient(querycfg)
208
+
})
204
209
205
210
got, err := provider.GetIngredients(t.Context(), "acmemarkets_806", "pinot", 1)
206
211
if err != nil {
+444
internal/brightdata/browser.go
+444
internal/brightdata/browser.go
···
1
+
package brightdata
2
+
3
+
import (
4
+
"context"
5
+
"encoding/base64"
6
+
"encoding/json"
7
+
"errors"
8
+
"fmt"
9
+
"math"
10
+
"net"
11
+
"net/http"
12
+
"net/url"
13
+
"strings"
14
+
"sync"
15
+
"sync/atomic"
16
+
"time"
17
+
18
+
"github.com/gobwas/ws"
19
+
"github.com/gobwas/ws/wsutil"
20
+
)
21
+
22
+
const (
23
+
DefaultBrowserWSEndpoint = "wss://brd.superproxy.io:9222"
24
+
defaultBrowserWait = 5 * time.Second
25
+
)
26
+
27
+
type BrowserClient struct {
28
+
wsEndpoint string
29
+
authHeader string
30
+
}
31
+
32
+
type BrowserClientConfig struct {
33
+
WSEndpoint string
34
+
}
35
+
36
+
type BrowserOptions struct {
37
+
WaitAfterNavigation time.Duration
38
+
}
39
+
40
+
type BrowserCookie struct {
41
+
Name string
42
+
Value string
43
+
Domain string
44
+
Path string
45
+
Expires *time.Time
46
+
HTTPOnly bool
47
+
Secure bool
48
+
Session bool
49
+
}
50
+
51
+
type cdpMessage struct {
52
+
ID int64 `json:"id,omitempty"`
53
+
Method string `json:"method,omitempty"`
54
+
Params json.RawMessage `json:"params,omitempty"`
55
+
Result json.RawMessage `json:"result,omitempty"`
56
+
SessionID string `json:"sessionId,omitempty"`
57
+
Error *cdpError `json:"error,omitempty"`
58
+
}
59
+
60
+
type cdpError struct {
61
+
Code int `json:"code"`
62
+
Message string `json:"message"`
63
+
}
64
+
65
+
type cdpClient struct {
66
+
conn net.Conn
67
+
68
+
writeMu sync.Mutex
69
+
nextID atomic.Int64
70
+
71
+
pendingMu sync.Mutex
72
+
pending map[int64]chan cdpMessage
73
+
events chan cdpMessage
74
+
readErr chan error
75
+
}
76
+
77
+
func NewBrowserClient(cfg BrowserClientConfig) (*BrowserClient, error) {
78
+
wsEndpoint, authHeader, err := browserWSEndpoint(cfg.WSEndpoint)
79
+
if err != nil {
80
+
return nil, err
81
+
}
82
+
return &BrowserClient{
83
+
wsEndpoint: wsEndpoint,
84
+
authHeader: authHeader,
85
+
}, nil
86
+
}
87
+
88
+
func (c *BrowserClient) Cookies(ctx context.Context, targetURL string, opts BrowserOptions) ([]BrowserCookie, error) {
89
+
targetURL = strings.TrimSpace(targetURL)
90
+
if targetURL == "" {
91
+
return nil, errors.New("target URL is required")
92
+
}
93
+
94
+
wait := opts.WaitAfterNavigation
95
+
if wait <= 0 {
96
+
wait = defaultBrowserWait
97
+
}
98
+
99
+
client, err := newCDPClient(ctx, c.wsEndpoint, c.authHeader)
100
+
if err != nil {
101
+
return nil, fmt.Errorf("dial browser websocket: %w", err)
102
+
}
103
+
defer func() {
104
+
_ = client.Close()
105
+
}()
106
+
107
+
// Bright Data rejects creating a target with the destination URL directly,
108
+
// so we create a blank tab first and drive navigation over the attached CDP session.
109
+
targetID, err := client.createTarget(ctx)
110
+
if err != nil {
111
+
return nil, fmt.Errorf("create browser target: %w", err)
112
+
}
113
+
defer func() {
114
+
closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
115
+
defer cancel()
116
+
_ = client.closeTarget(closeCtx, targetID)
117
+
}()
118
+
119
+
sessionID, err := client.attachToTarget(ctx, targetID)
120
+
if err != nil {
121
+
return nil, fmt.Errorf("attach to browser target: %w", err)
122
+
}
123
+
124
+
// Enable the minimum CDP domains we need before navigating so cookie state
125
+
// is available from the target session once the page settles.
126
+
if err := client.call(ctx, sessionID, "Page.enable", nil, nil); err != nil {
127
+
return nil, fmt.Errorf("enable page domain: %w", err)
128
+
}
129
+
if err := client.call(ctx, sessionID, "Network.enable", nil, nil); err != nil {
130
+
return nil, fmt.Errorf("enable network domain: %w", err)
131
+
}
132
+
// Navigation has to happen after attach; Bright Data exposes a browser-level
133
+
// websocket and the actual page work occurs within the per-target session.
134
+
if err := client.navigate(ctx, sessionID, targetURL); err != nil {
135
+
return nil, fmt.Errorf("navigate browser target: %w", err)
136
+
}
137
+
138
+
if wait > 0 {
139
+
timer := time.NewTimer(wait)
140
+
defer timer.Stop()
141
+
select {
142
+
case <-ctx.Done():
143
+
return nil, ctx.Err()
144
+
case <-timer.C:
145
+
}
146
+
}
147
+
148
+
return client.getCookies(ctx, sessionID, targetURL)
149
+
}
150
+
151
+
func CookieNamed(cookies []BrowserCookie, name string) (BrowserCookie, bool) {
152
+
name = strings.TrimSpace(name)
153
+
for _, cookie := range cookies {
154
+
if cookie.Name == name {
155
+
return cookie, true
156
+
}
157
+
}
158
+
return BrowserCookie{}, false
159
+
}
160
+
161
+
func browserWSEndpoint(rawEndpoint string) (string, string, error) {
162
+
endpoint := strings.TrimSpace(rawEndpoint)
163
+
if endpoint == "" {
164
+
endpoint = DefaultBrowserWSEndpoint
165
+
}
166
+
167
+
parsed, err := url.Parse(endpoint)
168
+
if err != nil {
169
+
return "", "", fmt.Errorf("parse Bright Data browser endpoint: %w", err)
170
+
}
171
+
if parsed.Scheme == "" || parsed.Host == "" {
172
+
return "", "", errors.New("bright data browser endpoint must be an absolute websocket URL")
173
+
}
174
+
if parsed.Scheme != "ws" && parsed.Scheme != "wss" {
175
+
return "", "", errors.New("bright data browser endpoint must use ws or wss")
176
+
}
177
+
178
+
headerValue, err := browserAuthHeader(parsed)
179
+
if err != nil {
180
+
return "", "", err
181
+
}
182
+
183
+
parsed.User = nil
184
+
if parsed.Path == "" {
185
+
parsed.Path = "/"
186
+
}
187
+
return parsed.String(), headerValue, nil
188
+
}
189
+
190
+
func browserAuthHeader(parsed *url.URL) (string, error) {
191
+
if parsed == nil {
192
+
return "", errors.New("browser endpoint is required")
193
+
}
194
+
195
+
user := ""
196
+
pass := ""
197
+
if parsed.User != nil {
198
+
user = parsed.User.Username()
199
+
pass, _ = parsed.User.Password()
200
+
}
201
+
if user == "" || pass == "" {
202
+
return "", errors.New("bright data browser endpoint must include USER:PASS credentials")
203
+
}
204
+
205
+
token := base64.StdEncoding.EncodeToString([]byte(user + ":" + pass))
206
+
return "Basic " + token, nil
207
+
}
208
+
209
+
func expiresAt(unixSeconds float64) *time.Time {
210
+
if unixSeconds <= 0 || math.IsNaN(unixSeconds) || math.IsInf(unixSeconds, 0) {
211
+
return nil
212
+
}
213
+
expires := time.Unix(0, int64(unixSeconds*float64(time.Second))).UTC()
214
+
return &expires
215
+
}
216
+
217
+
func newCDPClient(ctx context.Context, wsEndpoint, authHeader string) (*cdpClient, error) {
218
+
dialer := ws.Dialer{
219
+
Header: ws.HandshakeHeaderHTTP(http.Header{
220
+
"Authorization": []string{authHeader},
221
+
}),
222
+
}
223
+
conn, br, _, err := dialer.Dial(ctx, wsEndpoint)
224
+
if err != nil {
225
+
return nil, err
226
+
}
227
+
if br != nil {
228
+
_ = conn.Close()
229
+
return nil, errors.New("unexpected buffered websocket reader")
230
+
}
231
+
232
+
client := &cdpClient{
233
+
conn: conn,
234
+
pending: make(map[int64]chan cdpMessage),
235
+
events: make(chan cdpMessage, 64),
236
+
readErr: make(chan error, 1),
237
+
}
238
+
go client.readLoop()
239
+
return client, nil
240
+
}
241
+
242
+
func (c *cdpClient) Close() error {
243
+
return c.conn.Close()
244
+
}
245
+
246
+
func (c *cdpClient) createTarget(ctx context.Context) (string, error) {
247
+
var result struct {
248
+
TargetID string `json:"targetId"`
249
+
}
250
+
if err := c.call(ctx, "", "Target.createTarget", map[string]any{"url": "about:blank"}, &result); err != nil {
251
+
return "", err
252
+
}
253
+
if result.TargetID == "" {
254
+
return "", errors.New("browser target ID missing from response")
255
+
}
256
+
return result.TargetID, nil
257
+
}
258
+
259
+
func (c *cdpClient) attachToTarget(ctx context.Context, targetID string) (string, error) {
260
+
var result struct {
261
+
SessionID string `json:"sessionId"`
262
+
}
263
+
if err := c.call(ctx, "", "Target.attachToTarget", map[string]any{
264
+
"targetId": targetID,
265
+
"flatten": true,
266
+
}, &result); err != nil {
267
+
return "", err
268
+
}
269
+
if result.SessionID == "" {
270
+
return "", errors.New("browser session ID missing from response")
271
+
}
272
+
return result.SessionID, nil
273
+
}
274
+
275
+
func (c *cdpClient) navigate(ctx context.Context, sessionID, targetURL string) error {
276
+
var result struct {
277
+
FrameID string `json:"frameId"`
278
+
}
279
+
if err := c.call(ctx, sessionID, "Page.navigate", map[string]any{
280
+
"url": targetURL,
281
+
}, &result); err != nil {
282
+
return err
283
+
}
284
+
if result.FrameID == "" {
285
+
return errors.New("navigation frame ID missing from response")
286
+
}
287
+
return nil
288
+
}
289
+
290
+
func (c *cdpClient) closeTarget(ctx context.Context, targetID string) error {
291
+
var result struct {
292
+
Success bool `json:"success"`
293
+
}
294
+
if err := c.call(ctx, "", "Target.closeTarget", map[string]any{"targetId": targetID}, &result); err != nil {
295
+
return err
296
+
}
297
+
if !result.Success {
298
+
return errors.New("browser target close was not acknowledged")
299
+
}
300
+
return nil
301
+
}
302
+
303
+
func (c *cdpClient) getCookies(ctx context.Context, sessionID, targetURL string) ([]BrowserCookie, error) {
304
+
var result struct {
305
+
Cookies []struct {
306
+
Name string `json:"name"`
307
+
Value string `json:"value"`
308
+
Domain string `json:"domain"`
309
+
Path string `json:"path"`
310
+
Expires float64 `json:"expires"`
311
+
HTTPOnly bool `json:"httpOnly"`
312
+
Secure bool `json:"secure"`
313
+
Session bool `json:"session"`
314
+
} `json:"cookies"`
315
+
}
316
+
if err := c.call(ctx, sessionID, "Network.getCookies", map[string]any{
317
+
"urls": []string{targetURL},
318
+
}, &result); err != nil {
319
+
return nil, err
320
+
}
321
+
322
+
cookies := make([]BrowserCookie, 0, len(result.Cookies))
323
+
for _, cookie := range result.Cookies {
324
+
cookies = append(cookies, BrowserCookie{
325
+
Name: cookie.Name,
326
+
Value: cookie.Value,
327
+
Domain: cookie.Domain,
328
+
Path: cookie.Path,
329
+
Expires: expiresAt(cookie.Expires),
330
+
HTTPOnly: cookie.HTTPOnly,
331
+
Secure: cookie.Secure,
332
+
Session: cookie.Session,
333
+
})
334
+
}
335
+
return cookies, nil
336
+
}
337
+
338
+
func (c *cdpClient) call(ctx context.Context, sessionID, method string, params any, out any) error {
339
+
id := c.nextID.Add(1)
340
+
respCh := make(chan cdpMessage, 1)
341
+
342
+
c.pendingMu.Lock()
343
+
c.pending[id] = respCh
344
+
c.pendingMu.Unlock()
345
+
defer func() {
346
+
c.pendingMu.Lock()
347
+
delete(c.pending, id)
348
+
c.pendingMu.Unlock()
349
+
}()
350
+
351
+
msg := map[string]any{
352
+
"id": id,
353
+
"method": method,
354
+
}
355
+
if sessionID != "" {
356
+
msg["sessionId"] = sessionID
357
+
}
358
+
if params != nil {
359
+
msg["params"] = params
360
+
}
361
+
362
+
payload, err := json.Marshal(msg)
363
+
if err != nil {
364
+
return fmt.Errorf("marshal %s request: %w", method, err)
365
+
}
366
+
367
+
c.writeMu.Lock()
368
+
err = wsutil.WriteClientText(c.conn, payload)
369
+
c.writeMu.Unlock()
370
+
if err != nil {
371
+
return fmt.Errorf("send %s request: %w", method, err)
372
+
}
373
+
374
+
select {
375
+
case <-ctx.Done():
376
+
return ctx.Err()
377
+
case err := <-c.readErr:
378
+
return err
379
+
case resp := <-respCh:
380
+
if resp.Error != nil {
381
+
return fmt.Errorf("%s: %s (%d)", method, resp.Error.Message, resp.Error.Code)
382
+
}
383
+
if out == nil || len(resp.Result) == 0 {
384
+
return nil
385
+
}
386
+
if err := json.Unmarshal(resp.Result, out); err != nil {
387
+
return fmt.Errorf("decode %s response: %w", method, err)
388
+
}
389
+
return nil
390
+
}
391
+
}
392
+
393
+
func (c *cdpClient) readLoop() {
394
+
for {
395
+
payload, op, err := wsutil.ReadServerData(c.conn)
396
+
if err != nil {
397
+
c.failPending(err)
398
+
return
399
+
}
400
+
if op != ws.OpText {
401
+
continue
402
+
}
403
+
404
+
var msg cdpMessage
405
+
if err := json.Unmarshal(payload, &msg); err != nil {
406
+
c.failPending(fmt.Errorf("decode websocket message: %w", err))
407
+
return
408
+
}
409
+
410
+
if msg.ID == 0 {
411
+
select {
412
+
case c.events <- msg:
413
+
default:
414
+
}
415
+
continue
416
+
}
417
+
418
+
c.pendingMu.Lock()
419
+
respCh := c.pending[msg.ID]
420
+
c.pendingMu.Unlock()
421
+
if respCh == nil {
422
+
continue
423
+
}
424
+
respCh <- msg
425
+
}
426
+
}
427
+
428
+
func (c *cdpClient) failPending(err error) {
429
+
c.pendingMu.Lock()
430
+
defer c.pendingMu.Unlock()
431
+
432
+
select {
433
+
case c.readErr <- err:
434
+
default:
435
+
}
436
+
437
+
for id, respCh := range c.pending {
438
+
delete(c.pending, id)
439
+
select {
440
+
case respCh <- cdpMessage{Error: &cdpError{Message: err.Error()}}:
441
+
default:
442
+
}
443
+
}
444
+
}
+38
internal/brightdata/browser_test.go
+38
internal/brightdata/browser_test.go
···
1
+
package brightdata
2
+
3
+
import (
4
+
"encoding/base64"
5
+
"net/url"
6
+
"testing"
7
+
)
8
+
9
+
func TestBrowserWSEndpointUsesCredentialsFromURL(t *testing.T) {
10
+
t.Parallel()
11
+
12
+
wsEndpoint, authHeader, err := browserWSEndpoint("wss://user:pass@brd.superproxy.io:9222")
13
+
if err != nil {
14
+
t.Fatalf("browserWSEndpoint returned error: %v", err)
15
+
}
16
+
17
+
if wsEndpoint != "wss://brd.superproxy.io:9222/" {
18
+
t.Fatalf("unexpected endpoint: %q", wsEndpoint)
19
+
}
20
+
21
+
wantHeader := "Basic " + base64.StdEncoding.EncodeToString([]byte("user:pass"))
22
+
if authHeader != wantHeader {
23
+
t.Fatalf("unexpected auth header: %q", authHeader)
24
+
}
25
+
}
26
+
27
+
func TestBrowserAuthHeaderRejectsMissingPassword(t *testing.T) {
28
+
t.Parallel()
29
+
30
+
parsed, err := url.Parse("wss://brd.superproxy.io:9222")
31
+
if err != nil {
32
+
t.Fatalf("url.Parse returned error: %v", err)
33
+
}
34
+
35
+
if _, err := browserAuthHeader(parsed); err == nil {
36
+
t.Fatal("expected error for missing websocket credentials")
37
+
}
38
+
}
+5
-4
internal/config/config.go
+5
-4
internal/config/config.go
···
78
78
}
79
79
80
80
type AlbertsonsConfig struct {
81
-
Enable bool `json:"enable"`
81
+
Enable bool `json:"enable"`
82
+
// removing this keeps stores but disables inventory
82
83
SearchSubscriptionKey string `json:"search_subscription_key"`
83
-
SearchReese84 string `json:"search_reese84"`
84
+
// SearchReese84 string `json:"search_reese84"`
84
85
}
85
86
86
87
func (c *AlbertsonsConfig) IsEnabled() bool {
···
89
90
90
91
// can we dynamically disable if our key expires
91
92
func (c *AlbertsonsConfig) HasInventory() bool {
92
-
return c.SearchReese84 != ""
93
+
return c.SearchSubscriptionKey != ""
93
94
}
94
95
95
96
type PublixConfig struct {
···
175
176
Albertsons: AlbertsonsConfig{
176
177
Enable: envEnabled("ALBERTSONS_ENABLE"),
177
178
SearchSubscriptionKey: os.Getenv("ALBERTSONS_SEARCH_SUBSCRIPTION_KEY"),
178
-
SearchReese84: os.Getenv("ALBERTSONS_SEARCH_REESE84"),
179
+
// SearchReese84: os.Getenv("ALBERTSONS_SEARCH_REESE84"),
179
180
},
180
181
Publix: PublixConfig{
181
182
Enable: envEnabled("PUBLIX_ENABLE"),
-3
internal/config/config_test.go
-3
internal/config/config_test.go
···
88
88
if got, want := cfg.Albertsons.SearchSubscriptionKey, "sub-key"; got != want {
89
89
t.Fatalf("expected Albertsons subscription key %q, got %q", want, got)
90
90
}
91
-
if got, want := cfg.Albertsons.SearchReese84, "cookie-value"; got != want {
92
-
t.Fatalf("expected Albertsons reese84 %q, got %q", want, got)
93
-
}
94
91
}
95
92
96
93
func TestLoadReadsBrightDataProxyConfig(t *testing.T) {
+7
-1
internal/recipes/staples.go
+7
-1
internal/recipes/staples.go
···
94
94
return nil, fmt.Errorf("create bright data proxy-aware client: %w", err)
95
95
}
96
96
97
+
// only returns an err because it ensures a cache for reese84 tokens.
98
+
albertsonsProvider, err := albertsons.NewStaplesProvider(cfg.Albertsons, httpClient)
99
+
if err != nil {
100
+
return nil, fmt.Errorf("create albertsons staples provider: %w", err)
101
+
}
102
+
97
103
return []backendStaplesProvider{
98
104
kroger.NewStaplesProvider(krogerClient),
105
+
albertsonsProvider,
99
106
// actowiz.NewStaplesProvider(),
100
107
walmart.NewStaplesProvider(),
101
108
wholefoods.NewStaplesProvider(wholefoods.NewClient(httpClient)),
102
-
albertsons.NewStaplesProvider(cfg.Albertsons, httpClient),
103
109
}, nil
104
110
}
105
111