ai cooking
Ziplookup (#419)
* getting closer
* fumpt
* fumpt
* a little better
* remove redundant
* move to a hydrator and just give a laoder to it
* fumpt and missed hydrator
* lets go
* map with errors
* test for map with errrors
* sigh fumpt
* close readers
---------
Co-authored-by: paul miller <paul.miller>
authored by
Paul Miller
paul miller
and committed by
GitHub
b7509cad
4f470073
+964
-375
+15
-1
cmd/albertsons/main.go
+15
-1
cmd/albertsons/main.go
···
13
13
14
14
"careme/internal/albertsons"
15
15
"careme/internal/cache"
16
+
"careme/internal/locations"
16
17
"careme/internal/logsetup"
17
18
)
18
19
···
48
49
httpClient := &http.Client{Timeout: time.Duration(timeoutSec) * time.Second}
49
50
delay := time.Duration(delayMS) * time.Millisecond
50
51
52
+
synced, err := syncChains(ctx, cacheStore, httpClient, chains, delay)
53
+
if err != nil {
54
+
log.Fatalf("failed to sync Albertsons-family store summaries: %v", err)
55
+
}
56
+
57
+
fmt.Printf("synced %d Albertsons-family store summaries\n", synced)
58
+
}
59
+
60
+
func syncChains(ctx context.Context, cacheStore cache.ListCache, httpClient *http.Client, chains []albertsons.Chain, delay time.Duration) (int, error) {
51
61
var synced int
52
62
for _, chain := range chains {
53
63
chainSynced, err := syncChainFromSitemap(ctx, cacheStore, httpClient, chain, chain.SitemapURL(), delay)
···
58
68
synced += chainSynced
59
69
}
60
70
61
-
fmt.Printf("synced %d Albertsons-family store summaries\n", synced)
71
+
if err := albertsons.RebuildLocationIndex(ctx, cacheStore, locations.LoadCentroids()); err != nil {
72
+
return synced, fmt.Errorf("rebuild location index: %w", err)
73
+
}
74
+
75
+
return synced, nil
62
76
}
63
77
64
78
// not concurrent safe because url map is shared. Could fix that with etags or seperate maps.
+10
-3
cmd/albertsons/main_test.go
+10
-3
cmd/albertsons/main_test.go
···
41
41
}
42
42
}
43
43
44
-
func TestSyncChainFromSitemapSkipsKnownURLsWithCachedSummaries(t *testing.T) {
44
+
func TestSyncChainsSkipsKnownURLsWithCachedSummaries(t *testing.T) {
45
45
t.Parallel()
46
46
47
47
cacheStore := cache.NewInMemoryCache()
···
93
93
IDPrefix: "albertsons_",
94
94
}
95
95
96
-
synced, err := syncChainFromSitemap(context.Background(), cacheStore, httpClient, chain, baseURL+"/sitemap.xml", 0*time.Millisecond)
96
+
synced, err := syncChains(context.Background(), cacheStore, httpClient, []albertsons.Chain{chain}, 0*time.Millisecond)
97
97
if err != nil {
98
-
t.Fatalf("syncChainFromSitemap returned error: %v", err)
98
+
t.Fatalf("syncChains returned error: %v", err)
99
99
}
100
100
if synced != 0 {
101
101
t.Fatalf("expected 0 synced summaries, got %d", synced)
102
102
}
103
103
if pageRequests.Load() != 0 {
104
104
t.Fatalf("expected no page requests for cached url, got %d", pageRequests.Load())
105
+
}
106
+
exists, err := cacheStore.Exists(context.Background(), albertsons.LocationIndexCacheKey)
107
+
if err != nil {
108
+
t.Fatalf("expected compact location index: %v", err)
109
+
}
110
+
if !exists {
111
+
t.Fatal("expected compact location index to exist")
105
112
}
106
113
}
107
114
+6
-1
cmd/aldi/main.go
+6
-1
cmd/aldi/main.go
···
11
11
12
12
"careme/internal/aldi"
13
13
"careme/internal/cache"
14
+
"careme/internal/locations"
14
15
"careme/internal/logsetup"
15
16
)
16
17
···
53
54
fmt.Printf("synced %d ALDI store summaries\n", synced)
54
55
}
55
56
56
-
func syncLocations(ctx context.Context, cacheStore cache.Cache, client summaryClient) (int, error) {
57
+
func syncLocations(ctx context.Context, cacheStore cache.ListCache, client summaryClient) (int, error) {
57
58
summaries, err := client.StoreSummaries(ctx)
58
59
if err != nil {
59
60
return 0, err
···
66
67
continue
67
68
}
68
69
synced++
70
+
}
71
+
72
+
if err := aldi.RebuildLocationIndex(ctx, cacheStore, locations.LoadCentroids()); err != nil {
73
+
return synced, err
69
74
}
70
75
return synced, nil
71
76
}
+7
cmd/aldi/main_test.go
+7
cmd/aldi/main_test.go
···
42
42
if len(keys) != 1 || keys[0] != "aldi_F100" {
43
43
t.Fatalf("unexpected cached keys: %v", keys)
44
44
}
45
+
exists, err := cacheStore.Exists(context.Background(), aldi.LocationIndexCacheKey)
46
+
if err != nil {
47
+
t.Fatalf("expected compact location index: %v", err)
48
+
}
49
+
if !exists {
50
+
t.Fatal("expected compact location index to exist")
51
+
}
45
52
}
46
53
47
54
type fakeSummaryClient struct {
+4
cmd/heb/main.go
+4
cmd/heb/main.go
···
12
12
13
13
"careme/internal/cache"
14
14
"careme/internal/heb"
15
+
"careme/internal/locations"
15
16
"careme/internal/sitemapfetch"
16
17
)
17
18
···
92
93
if err := heb.SaveStoreURLMap(ctx, cacheStore, urlMap); err != nil {
93
94
return synced, err
94
95
}
96
+
}
97
+
if err := heb.RebuildLocationIndex(ctx, cacheStore, locations.LoadCentroids()); err != nil {
98
+
return synced, err
95
99
}
96
100
return synced, nil
97
101
}
+7
cmd/heb/main_test.go
+7
cmd/heb/main_test.go
···
68
68
if pageRequests.Load() != 0 {
69
69
t.Fatalf("expected no page requests for cached url, got %d", pageRequests.Load())
70
70
}
71
+
exists, err := cacheStore.Exists(context.Background(), heb.LocationIndexCacheKey)
72
+
if err != nil {
73
+
t.Fatalf("expected compact location index: %v", err)
74
+
}
75
+
if !exists {
76
+
t.Fatal("expected compact location index to exist")
77
+
}
71
78
}
72
79
73
80
func TestSyncFromSitemapAddsNewURLMappings(t *testing.T) {
+5
cmd/publix/main.go
+5
cmd/publix/main.go
···
12
12
"time"
13
13
14
14
"careme/internal/cache"
15
+
"careme/internal/locations"
15
16
"careme/internal/logsetup"
16
17
"careme/internal/publix"
17
18
)
···
165
166
if err := publix.SaveMissingStoreIDs(ctx, cacheStore, missingStoreIDs); err != nil {
166
167
return stats, err
167
168
}
169
+
}
170
+
171
+
if err := publix.RebuildLocationIndex(ctx, cacheStore, locations.LoadCentroids()); err != nil {
172
+
return stats, err
168
173
}
169
174
170
175
return stats, nil
+7
cmd/publix/main_test.go
+7
cmd/publix/main_test.go
···
56
56
if _, ok := missingIDs["1084"]; !ok {
57
57
t.Fatalf("expected missing store id 1084 to be cached")
58
58
}
59
+
exists, err := cacheStore.Exists(context.Background(), publix.LocationIndexCacheKey)
60
+
if err != nil {
61
+
t.Fatalf("expected compact location index: %v", err)
62
+
}
63
+
if !exists {
64
+
t.Fatal("expected compact location index to exist")
65
+
}
59
66
}
60
67
61
68
func TestSyncStoresSkipsKnownMissingAndCachedSuccesses(t *testing.T) {
+5
cmd/wegmans/main.go
+5
cmd/wegmans/main.go
···
12
12
"time"
13
13
14
14
"careme/internal/cache"
15
+
"careme/internal/locations"
15
16
"careme/internal/logsetup"
16
17
"careme/internal/wegmans"
17
18
)
···
128
129
if cfg.delay > 0 && storeNumber < cfg.endID {
129
130
time.Sleep(cfg.delay)
130
131
}
132
+
}
133
+
134
+
if err := wegmans.RebuildLocationIndex(ctx, cacheStore, locations.LoadCentroids()); err != nil {
135
+
return stats, err
131
136
}
132
137
133
138
return stats, nil
+7
cmd/wegmans/main_test.go
+7
cmd/wegmans/main_test.go
···
48
48
if len(keys) != 1 || keys[0] != "1" {
49
49
t.Fatalf("unexpected cached keys: %v", keys)
50
50
}
51
+
exists, err := cacheStore.Exists(context.Background(), wegmans.LocationIndexCacheKey)
52
+
if err != nil {
53
+
t.Fatalf("expected compact location index: %v", err)
54
+
}
55
+
if !exists {
56
+
t.Fatal("expected compact location index to exist")
57
+
}
51
58
}
52
59
53
60
func TestSyncStoresSkipsCachedSummaries(t *testing.T) {
+5
cmd/wholefoods/main.go
+5
cmd/wholefoods/main.go
···
11
11
"time"
12
12
13
13
"careme/internal/cache"
14
+
"careme/internal/locations"
14
15
"careme/internal/logsetup"
15
16
"careme/internal/wholefoods"
16
17
)
···
64
65
}
65
66
time.Sleep(5 * time.Second) // be nice to the server no rush here
66
67
synced++
68
+
}
69
+
70
+
if err := wholefoods.RebuildLocationIndex(ctx, cacheStore, locations.LoadCentroids()); err != nil {
71
+
log.Fatalf("failed to rebuild Whole Foods location index: %v", err)
67
72
}
68
73
69
74
fmt.Printf("synced %d Whole Foods store summaries\n", synced)
+6
docs/cache-layout.md
+6
docs/cache-layout.md
···
40
40
| `location-store-requests/` | JSON `{store_id, zip, requested_at}` for stores present in location search but not yet supported for staples | `internal/locations/locations.go` (`POST /locations/request-store`) | `internal/locations/locations.go` (`RequestedStoreIDs`) and operational triage from shared cache/blob storage |
41
41
| `aldi/stores/` | JSON `aldi.StoreSummary` keyed by prefixed ALDI location ID | `cmd/aldi` and `internal/aldi` cache helpers | `internal/aldi` location backend |
42
42
| `albertsons/stores/` | JSON `albertsons.StoreSummary` keyed by prefixed Albertsons-family location ID | `cmd/albertsons` and `internal/albertsons` cache helpers | `internal/albertsons` location backend |
43
+
| `albertsons/store_locations.json` | JSON `[]storeindex.Entry` spatial index for Albertsons-family stores (`id`, `lat`, `lon`) | `cmd/albertsons` rebuilds after sync | `internal/albertsons` location backend |
43
44
| `albertsons/store_url_map.json` | JSON object mapping store URL to prefixed Albertsons-family location ID | `cmd/albertsons` and `internal/albertsons` cache helpers | `cmd/albertsons` incremental sync |
45
+
| `aldi/store_locations.json` | JSON `[]storeindex.Entry` spatial index for ALDI stores (`id`, `lat`, `lon`) | `cmd/aldi` rebuilds after sync | `internal/aldi` location backend |
44
46
| `heb/stores/` | JSON `heb.StoreSummary` keyed by prefixed HEB location ID | `cmd/heb` and `internal/heb` cache helpers | `internal/heb` location backend |
47
+
| `heb/store_locations.json` | JSON `[]storeindex.Entry` spatial index for HEB stores (`id`, `lat`, `lon`) | `cmd/heb` rebuilds after sync | `internal/heb` location backend |
45
48
| `heb/store_url_map.json` | JSON object mapping store URL to prefixed HEB location ID | `cmd/heb` and `internal/heb` cache helpers | `cmd/heb` incremental sync |
46
49
| `publix/stores/` | JSON `publix.StoreSummary` keyed by numeric Publix store ID | `cmd/publix` and `internal/publix` cache helpers | `internal/publix` location backend |
50
+
| `publix/store_locations.json` | JSON `[]storeindex.Entry` spatial index for Publix stores (`id`, `lat`, `lon`) | `cmd/publix` rebuilds after sync | `internal/publix` location backend |
47
51
| `publix/store_url_map.json` | JSON object mapping numeric Publix store ID to canonical location URL | `cmd/publix` and `internal/publix` cache helpers | `cmd/publix` incremental sync |
48
52
| `publix/missing_store_ids.json` | JSON array of numeric Publix store IDs known to redirect back to `/locations` | `cmd/publix` and `internal/publix` cache helpers | `cmd/publix` incremental sync |
49
53
| `wegmans/stores/` | JSON `wegmans.StoreSummary` keyed by numeric Wegmans store ID | `cmd/wegmans` and `internal/wegmans` cache helpers | `internal/wegmans` location backend |
54
+
| `wegmans/store_locations.json` | JSON `[]storeindex.Entry` spatial index for Wegmans stores (`id`, `lat`, `lon`) | `cmd/wegmans` rebuilds after sync | `internal/wegmans` location backend |
50
55
| `wholefoods/stores/` | JSON `wholefoods.StoreSummaryResponse` keyed by Whole Foods store ID | `cmd/wholefoods` and `internal/wholefoods` cache helpers | `internal/wholefoods` location backend |
56
+
| `wholefoods/store_locations.json` | JSON `[]storeindex.Entry` spatial index for Whole Foods stores (`id`, `lat`, `lon`) | `cmd/wholefoods` rebuilds after sync | `internal/wholefoods` location backend |
51
57
| `wholefoods/store_url_map.json` | JSON object mapping store URL to Whole Foods store ID | `cmd/wholefoods` and `internal/wholefoods` cache helpers | `cmd/wholefoods` when `-stores` is not provided |
52
58
53
59
## Notes
+30
-39
internal/albertsons/cache.go
+30
-39
internal/albertsons/cache.go
···
5
5
"encoding/json"
6
6
"errors"
7
7
"fmt"
8
-
"log/slog"
9
8
10
9
"careme/internal/cache"
10
+
"careme/internal/locations/storeindex"
11
11
"careme/internal/sitemapfetch"
12
12
13
13
locationtypes "careme/internal/locations/types"
14
-
15
-
"github.com/samber/lo"
16
-
lop "github.com/samber/lo/parallel"
17
14
)
18
15
19
16
const (
20
-
Container = "albertsons"
21
-
StoreCachePrefix = "albertsons/stores/"
22
-
StoreURLMapCacheKey = "albertsons/store_url_map.json"
17
+
Container = "albertsons"
18
+
StoreCachePrefix = "albertsons/stores/"
19
+
StoreURLMapCacheKey = "albertsons/store_url_map.json"
20
+
LocationIndexCacheKey = "albertsons/store_locations.json"
23
21
)
24
22
25
23
func SaveStoreURLMap(ctx context.Context, c cache.Cache, urlMap map[string]string) error {
···
46
44
return nil
47
45
}
48
46
49
-
func loadCachedStoreSummaries(ctx context.Context, c cache.ListCache) ([]*StoreSummary, error) {
50
-
keys, err := c.List(ctx, StoreCachePrefix, "")
51
-
if err != nil {
52
-
return nil, fmt.Errorf("list cached store summaries: %w", err)
53
-
}
54
-
55
-
// expensive. Just save a smaller map of centroids
56
-
// see branch cachezipcodes. Maybe time for a real db instead of blob
57
-
summaries := lop.Map(keys, func(key string, _ int) *StoreSummary {
58
-
reader, err := c.Get(ctx, StoreCachePrefix+key)
59
-
if err != nil {
60
-
slog.WarnContext(ctx, "failed to read cached albertsons store summary", "key", key, "error", err)
61
-
return nil
47
+
func RebuildLocationIndex(ctx context.Context, c cache.ListCache, zipLookup storeindex.ZipCentroidLookup) error {
48
+
_, err := storeindex.RebuildFromStoreSummaries(ctx, c, StoreCachePrefix, LocationIndexCacheKey, func(summary StoreSummary) storeindex.Entry {
49
+
lat, lon := storeindex.Coordinates(summary.Lat, summary.Lon, summary.ZipCode, zipLookup)
50
+
return storeindex.Entry{
51
+
ID: summary.ID,
52
+
Lat: lat,
53
+
Lon: lon,
62
54
}
63
-
defer func() {
64
-
_ = reader.Close()
65
-
}()
55
+
})
56
+
return err
57
+
}
66
58
67
-
var summary StoreSummary
68
-
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
69
-
slog.WarnContext(ctx, "failed to decode cached albertsons store summary", "key", key, "error", err)
70
-
return nil
71
-
}
72
-
return &summary
73
-
})
59
+
type loader struct {
60
+
cache cache.Cache
61
+
}
74
62
75
-
summaries = lo.Compact(summaries)
76
-
if len(summaries) == 0 {
77
-
return nil, fmt.Errorf("failed to load albertsons locations")
63
+
func (l *loader) Load(ctx context.Context, locationID string) (locationtypes.Location, error) {
64
+
reader, err := l.cache.Get(ctx, StoreCachePrefix+locationID)
65
+
if err != nil {
66
+
return locationtypes.Location{}, err
78
67
}
79
-
slog.InfoContext(ctx, "loaded albertsons locations", "count", len(summaries))
68
+
defer func() {
69
+
_ = reader.Close()
70
+
}()
80
71
81
-
return summaries, nil
82
-
}
83
-
84
-
func storeSummaryToLocation(summary StoreSummary) locationtypes.Location {
72
+
var summary StoreSummary
73
+
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
74
+
return locationtypes.Location{}, fmt.Errorf("decode albertsons store summary: %w", err)
75
+
}
85
76
return locationtypes.Location{
86
77
ID: summary.ID,
87
78
Name: summary.Name,
···
91
82
Lat: summary.Lat,
92
83
Lon: summary.Lon,
93
84
Chain: Container,
94
-
}
85
+
}, nil
95
86
}
+18
-18
internal/albertsons/locations.go
+18
-18
internal/albertsons/locations.go
···
7
7
8
8
"careme/internal/cache"
9
9
"careme/internal/config"
10
+
"careme/internal/locations/hydrator"
10
11
"careme/internal/locations/nearby"
12
+
"careme/internal/locations/storeindex"
13
+
11
14
locationtypes "careme/internal/locations/types"
12
15
)
13
16
···
17
20
18
21
type LocationBackend struct {
19
22
zipLookup centroidByZip
20
-
byID map[string]locationtypes.Location
23
+
spatial []locationtypes.Location
24
+
hydrator *hydrator.LazyHydrator
21
25
}
22
26
23
27
func NewLocationBackendFromConfig(ctx context.Context, cfg *config.Config, zipLookup centroidByZip) (*LocationBackend, error) {
···
38
42
}
39
43
40
44
func newLocationBackend(ctx context.Context, c cache.ListCache, zipLookup centroidByZip) (*LocationBackend, error) {
41
-
summaries, err := loadCachedStoreSummaries(ctx, c)
45
+
entries, err := storeindex.Load(ctx, c, LocationIndexCacheKey)
42
46
if err != nil {
43
-
return nil, err
47
+
return nil, fmt.Errorf("load albertsons locations index: %w", err)
44
48
}
45
49
46
-
byID := make(map[string]locationtypes.Location, len(summaries))
47
-
for _, summary := range summaries {
48
-
loc := storeSummaryToLocation(*summary)
49
-
byID[loc.ID] = loc
50
+
spatial := make([]locationtypes.Location, 0, len(entries))
51
+
for _, entry := range entries {
52
+
spatial = append(spatial, entry.ToLocation())
50
53
}
51
54
52
55
return &LocationBackend{
53
56
zipLookup: zipLookup,
54
-
byID: byID,
57
+
spatial: spatial,
58
+
hydrator: hydrator.NewLazyHydrator(&loader{c}),
55
59
}, nil
56
60
}
57
61
···
63
67
return false
64
68
}
65
69
66
-
func (b *LocationBackend) GetLocationByID(_ context.Context, locationID string) (*locationtypes.Location, error) {
70
+
func (b *LocationBackend) GetLocationByID(ctx context.Context, locationID string) (*locationtypes.Location, error) {
67
71
locationID = strings.TrimSpace(locationID)
68
72
if !IsID(locationID) {
69
73
return nil, fmt.Errorf("albertsons location id %q is invalid", locationID)
70
74
}
71
75
72
-
loc, exists := b.byID[locationID]
73
-
if !exists {
74
-
return nil, fmt.Errorf("albertsons location %q not found", locationID)
76
+
loc, err := b.hydrator.Hydrate(ctx, locationID)
77
+
if err != nil {
78
+
return nil, err
75
79
}
76
-
77
80
copy := loc
78
81
return ©, nil
79
82
}
80
83
81
84
func (b *LocationBackend) GetLocationsByZip(ctx context.Context, zipcode string) ([]locationtypes.Location, error) {
82
-
candidates := make([]locationtypes.Location, 0, len(b.byID))
83
-
for _, loc := range b.byID {
84
-
candidates = append(candidates, loc)
85
-
}
86
-
return nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, candidates, nearby.MaxLocationDistanceMiles), nil
85
+
candidates := nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, b.spatial, nearby.MaxLocationDistanceMiles)
86
+
return storeindex.HydrateLocations(ctx, candidates, b.hydrator.Hydrate)
87
87
}
+88
-8
internal/albertsons/locations_test.go
+88
-8
internal/albertsons/locations_test.go
···
2
2
3
3
import (
4
4
"context"
5
+
"encoding/json"
6
+
"io"
5
7
"strings"
6
8
"testing"
7
9
···
16
18
if err := CacheStoreSummary(context.Background(), cacheStore, nearbySummary()); err != nil {
17
19
t.Fatalf("CacheStoreSummary returned error: %v", err)
18
20
}
19
-
20
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
21
+
zipLookup := staticZIPLookup{
21
22
"98006": {Lat: 47.5750, Lon: -122.1400},
22
-
})
23
+
}
24
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
25
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
26
+
}
27
+
28
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
23
29
if err != nil {
24
30
t.Fatalf("NewLocationBackend returned error: %v", err)
25
31
}
···
38
44
if loc.Name != "Safeway 15100 SE 38th St" || loc.ZipCode != "98006" || loc.Chain != "albertsons" {
39
45
t.Fatalf("unexpected location: %+v", loc)
40
46
}
47
+
reader, err := cacheStore.Get(context.Background(), LocationIndexCacheKey)
48
+
if err != nil {
49
+
t.Fatalf("expected compact location index to be cached: %v", err)
50
+
}
51
+
defer func() {
52
+
_ = reader.Close()
53
+
}()
54
+
55
+
raw, err := io.ReadAll(reader)
56
+
if err != nil {
57
+
t.Fatalf("ReadAll returned error: %v", err)
58
+
}
59
+
60
+
var entries []map[string]any
61
+
if err := json.Unmarshal(raw, &entries); err != nil {
62
+
t.Fatalf("Unmarshal returned error: %v", err)
63
+
}
64
+
if len(entries) != 1 {
65
+
t.Fatalf("expected 1 entry in compact location index, got %d", len(entries))
66
+
}
67
+
if _, ok := entries[0]["name"]; ok {
68
+
t.Fatalf("expected compact location index to omit hydrated fields, got %v", entries[0])
69
+
}
41
70
}
42
71
43
72
func TestLocationBackendGetLocationsByZipUsesDistance(t *testing.T) {
···
50
79
if err := CacheStoreSummary(context.Background(), cacheStore, farSummary()); err != nil {
51
80
t.Fatalf("cache far summary: %v", err)
52
81
}
53
-
54
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
82
+
zipLookup := staticZIPLookup{
55
83
"98006": {Lat: 47.5750, Lon: -122.1400},
56
-
})
84
+
}
85
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
86
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
87
+
}
88
+
89
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
57
90
if err != nil {
58
91
t.Fatalf("NewLocationBackend returned error: %v", err)
59
92
}
···
70
103
}
71
104
if locs[0].Chain != "albertsons" {
72
105
t.Fatalf("unexpected location chain: %q", locs[0].Chain)
106
+
}
107
+
if locs[0].Name != "Safeway 15100 SE 38th St" || locs[0].Address != "15100 SE 38th St" {
108
+
t.Fatalf("expected hydrated location details, got %+v", locs[0])
109
+
}
110
+
}
111
+
112
+
func TestLocationBackendUsesZipCentroidWhenCoordinatesMissing(t *testing.T) {
113
+
t.Parallel()
114
+
115
+
cacheStore := cache.NewInMemoryCache()
116
+
if err := CacheStoreSummary(context.Background(), cacheStore, noCoordsSummary()); err != nil {
117
+
t.Fatalf("cache no-coords summary: %v", err)
118
+
}
119
+
zipLookup := staticZIPLookup{
120
+
"98006": {Lat: 47.5750, Lon: -122.1400},
121
+
}
122
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
123
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
124
+
}
125
+
126
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
127
+
if err != nil {
128
+
t.Fatalf("NewLocationBackend returned error: %v", err)
129
+
}
130
+
131
+
locs, err := backend.GetLocationsByZip(context.Background(), "98006")
132
+
if err != nil {
133
+
t.Fatalf("GetLocationsByZip returned error: %v", err)
134
+
}
135
+
if len(locs) != 1 {
136
+
t.Fatalf("expected 1 nearby location, got %d", len(locs))
137
+
}
138
+
if locs[0].ID != "safeway_1444" {
139
+
t.Fatalf("unexpected location id: %q", locs[0].ID)
73
140
}
74
141
}
75
142
···
82
149
if err == nil {
83
150
t.Fatal("expected NewLocationBackend to return an error")
84
151
}
85
-
if !strings.Contains(err.Error(), "failed to load albertsons locations") {
86
-
t.Fatalf("expected missing summaries error, got %v", err)
152
+
if !strings.Contains(err.Error(), "load albertsons locations index") {
153
+
t.Fatalf("expected missing index error, got %v", err)
87
154
}
88
155
}
89
156
···
135
202
Lon: &lon,
136
203
}
137
204
}
205
+
206
+
func noCoordsSummary() *StoreSummary {
207
+
return &StoreSummary{
208
+
ID: "safeway_1444",
209
+
Brand: "safeway",
210
+
Domain: "local.safeway.com",
211
+
StoreID: "1444",
212
+
Name: "Safeway 15100 SE 38th St",
213
+
Address: "15100 SE 38th St",
214
+
State: "WA",
215
+
ZipCode: "98006",
216
+
}
217
+
}
+31
-35
internal/aldi/cache.go
+31
-35
internal/aldi/cache.go
···
5
5
"encoding/json"
6
6
"errors"
7
7
"fmt"
8
-
"log/slog"
9
8
10
9
"careme/internal/cache"
11
-
locationtypes "careme/internal/locations/types"
10
+
"careme/internal/locations/storeindex"
12
11
13
-
"github.com/samber/lo"
14
-
lop "github.com/samber/lo/parallel"
12
+
locationtypes "careme/internal/locations/types"
15
13
)
14
+
15
+
const LocationIndexCacheKey = "aldi/store_locations.json"
16
16
17
17
func CacheStoreSummary(ctx context.Context, c cache.Cache, summary *StoreSummary) error {
18
18
if summary == nil {
···
30
30
return nil
31
31
}
32
32
33
-
func loadCachedStoreSummaries(ctx context.Context, c cache.ListCache) ([]*StoreSummary, error) {
34
-
keys, err := c.List(ctx, StoreCachePrefix, "")
35
-
if err != nil {
36
-
return nil, fmt.Errorf("list cached store summaries: %w", err)
37
-
}
38
-
39
-
summaries := lop.Map(keys, func(key string, _ int) *StoreSummary {
40
-
reader, err := c.Get(ctx, StoreCachePrefix+key)
41
-
if err != nil {
42
-
slog.WarnContext(ctx, "failed to read cached ALDI store summary", "key", key, "error", err)
43
-
return nil
44
-
}
45
-
defer func() {
46
-
_ = reader.Close()
47
-
}()
33
+
func RebuildLocationIndex(ctx context.Context, c cache.ListCache, zipLookup storeindex.ZipCentroidLookup) error {
34
+
_, err := storeindex.RebuildFromStoreSummaries(ctx, c, StoreCachePrefix, LocationIndexCacheKey,
35
+
func(summary StoreSummary) storeindex.Entry {
36
+
lat, lon := storeindex.Coordinates(summary.Lat, summary.Lon, summary.ZipCode, zipLookup)
37
+
return storeindex.Entry{
38
+
ID: summary.ID,
39
+
Lat: lat,
40
+
Lon: lon,
41
+
}
42
+
})
43
+
return err
44
+
}
48
45
49
-
var summary StoreSummary
50
-
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
51
-
slog.WarnContext(ctx, "failed to decode cached ALDI store summary", "key", key, "error", err)
52
-
return nil
53
-
}
54
-
return &summary
55
-
})
46
+
type loader struct {
47
+
cache cache.Cache
48
+
}
56
49
57
-
summaries = lo.Compact(summaries)
58
-
if len(summaries) == 0 {
59
-
return nil, fmt.Errorf("failed to load aldi locations")
50
+
func (l *loader) Load(ctx context.Context, locationID string) (locationtypes.Location, error) {
51
+
reader, err := l.cache.Get(ctx, StoreCachePrefix+locationID)
52
+
if err != nil {
53
+
return locationtypes.Location{}, err
60
54
}
61
-
slog.InfoContext(ctx, "loaded ALDI locations", "count", len(summaries))
62
-
63
-
return summaries, nil
64
-
}
55
+
defer func() {
56
+
_ = reader.Close()
57
+
}()
65
58
66
-
func storeSummaryToLocation(summary StoreSummary) locationtypes.Location {
59
+
var summary StoreSummary
60
+
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
61
+
return locationtypes.Location{}, fmt.Errorf("decode ALDI store summary: %w", err)
62
+
}
67
63
return locationtypes.Location{
68
64
ID: summary.ID,
69
65
Name: summary.Name,
···
73
69
Lat: summary.Lat,
74
70
Lon: summary.Lon,
75
71
Chain: Container,
76
-
}
72
+
}, nil
77
73
}
+19
-19
internal/aldi/locations.go
+19
-19
internal/aldi/locations.go
···
7
7
8
8
"careme/internal/cache"
9
9
"careme/internal/config"
10
+
"careme/internal/locations/hydrator"
10
11
"careme/internal/locations/nearby"
12
+
"careme/internal/locations/storeindex"
13
+
11
14
locationtypes "careme/internal/locations/types"
12
15
)
13
16
···
17
20
18
21
type LocationBackend struct {
19
22
zipLookup centroidByZip
20
-
byID map[string]locationtypes.Location
23
+
spatial []locationtypes.Location
24
+
hydrator *hydrator.LazyHydrator
21
25
}
22
26
23
27
func NewLocationBackendFromConfig(ctx context.Context, cfg *config.Config, zipLookup centroidByZip) (*LocationBackend, error) {
···
35
39
return newLocationBackend(ctx, listCache, zipLookup)
36
40
}
37
41
38
-
func newLocationBackend(ctx context.Context, c cache.ListCache, zipLookup centroidByZip) (*LocationBackend, error) {
39
-
summaries, err := loadCachedStoreSummaries(ctx, c)
42
+
func newLocationBackend(ctx context.Context, c cache.Cache, zipLookup centroidByZip) (*LocationBackend, error) {
43
+
entries, err := storeindex.Load(ctx, c, LocationIndexCacheKey)
40
44
if err != nil {
41
-
return nil, err
45
+
return nil, fmt.Errorf("load aldi locations index: %w", err)
42
46
}
43
47
44
-
byID := make(map[string]locationtypes.Location, len(summaries))
45
-
for _, summary := range summaries {
46
-
loc := storeSummaryToLocation(*summary)
47
-
byID[loc.ID] = loc
48
+
spatial := make([]locationtypes.Location, 0, len(entries))
49
+
for _, entry := range entries {
50
+
spatial = append(spatial, entry.ToLocation())
48
51
}
49
52
50
53
return &LocationBackend{
51
54
zipLookup: zipLookup,
52
-
byID: byID,
55
+
spatial: spatial,
56
+
hydrator: hydrator.NewLazyHydrator(&loader{c}),
53
57
}, nil
54
58
}
55
59
···
61
65
return false
62
66
}
63
67
64
-
func (b *LocationBackend) GetLocationByID(_ context.Context, locationID string) (*locationtypes.Location, error) {
68
+
func (b *LocationBackend) GetLocationByID(ctx context.Context, locationID string) (*locationtypes.Location, error) {
65
69
locationID = strings.TrimSpace(locationID)
66
70
if !IsID(locationID) {
67
71
return nil, fmt.Errorf("ALDI location id %q is invalid", locationID)
68
72
}
69
73
70
-
loc, exists := b.byID[locationID]
71
-
if !exists {
72
-
return nil, fmt.Errorf("ALDI location %q not found", locationID)
74
+
loc, err := b.hydrator.Hydrate(ctx, locationID)
75
+
if err != nil {
76
+
return nil, err
73
77
}
74
-
75
78
copy := loc
76
79
return ©, nil
77
80
}
78
81
79
82
func (b *LocationBackend) GetLocationsByZip(ctx context.Context, zipcode string) ([]locationtypes.Location, error) {
80
-
candidates := make([]locationtypes.Location, 0, len(b.byID))
81
-
for _, loc := range b.byID {
82
-
candidates = append(candidates, loc)
83
-
}
84
-
return nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, candidates, nearby.MaxLocationDistanceMiles), nil
83
+
candidates := nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, b.spatial, nearby.MaxLocationDistanceMiles)
84
+
return storeindex.HydrateLocations(ctx, candidates, b.hydrator.Hydrate)
85
85
}
86
86
87
87
func IsID(locationID string) bool {
+21
-8
internal/aldi/locations_test.go
+21
-8
internal/aldi/locations_test.go
···
16
16
if err := CacheStoreSummary(context.Background(), cacheStore, nearbySummary()); err != nil {
17
17
t.Fatalf("CacheStoreSummary returned error: %v", err)
18
18
}
19
-
20
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
19
+
zipLookup := staticZIPLookup{
21
20
"60610": {Lat: 41.9033, Lon: -87.6313},
22
-
})
21
+
}
22
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
23
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
24
+
}
25
+
26
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
23
27
if err != nil {
24
28
t.Fatalf("newLocationBackend returned error: %v", err)
25
29
}
···
35
39
if loc.Name != "ALDI 201 W Division St" || loc.ZipCode != "60610" || loc.Chain != "aldi" {
36
40
t.Fatalf("unexpected location: %+v", loc)
37
41
}
42
+
reader, err := cacheStore.Get(context.Background(), LocationIndexCacheKey)
43
+
if err != nil {
44
+
t.Fatalf("expected compact location index to be cached: %v", err)
45
+
}
46
+
_ = reader.Close()
38
47
}
39
48
40
49
func TestLocationBackendGetLocationsByZipUsesDistance(t *testing.T) {
···
47
56
if err := CacheStoreSummary(context.Background(), cacheStore, farSummary()); err != nil {
48
57
t.Fatalf("cache far summary: %v", err)
49
58
}
50
-
51
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
59
+
zipLookup := staticZIPLookup{
52
60
"60610": {Lat: 41.9033, Lon: -87.6313},
53
-
})
61
+
}
62
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
63
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
64
+
}
65
+
66
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
54
67
if err != nil {
55
68
t.Fatalf("newLocationBackend returned error: %v", err)
56
69
}
···
79
92
if err == nil {
80
93
t.Fatal("expected newLocationBackend to return an error")
81
94
}
82
-
if !strings.Contains(err.Error(), "failed to load aldi locations") {
83
-
t.Fatalf("expected missing summaries error, got %v", err)
95
+
if !strings.Contains(err.Error(), "load aldi locations index") {
96
+
t.Fatalf("expected missing index error, got %v", err)
84
97
}
85
98
}
86
99
+28
-34
internal/heb/cache.go
+28
-34
internal/heb/cache.go
···
5
5
"encoding/json"
6
6
"errors"
7
7
"fmt"
8
-
"log/slog"
9
8
10
9
"careme/internal/cache"
11
-
locationtypes "careme/internal/locations/types"
10
+
"careme/internal/locations/storeindex"
12
11
"careme/internal/sitemapfetch"
13
12
14
-
"github.com/samber/lo"
15
-
lop "github.com/samber/lo/parallel"
13
+
locationtypes "careme/internal/locations/types"
16
14
)
17
15
18
16
const (
···
21
19
StoreURLMapCacheKey = "heb/store_url_map.json"
22
20
LocationIDPrefix = "heb_"
23
21
DefaultStoreSitemapURL = "https://www.heb.com/sitemap/storeSitemap.xml"
22
+
LocationIndexCacheKey = "heb/store_locations.json"
24
23
)
25
24
26
25
func SaveStoreURLMap(ctx context.Context, c cache.Cache, urlMap map[string]string) error {
···
47
46
return nil
48
47
}
49
48
50
-
func loadCachedStoreSummaries(ctx context.Context, c cache.ListCache) ([]*StoreSummary, error) {
51
-
keys, err := c.List(ctx, StoreCachePrefix, "")
52
-
if err != nil {
53
-
return nil, fmt.Errorf("list cached store summaries: %w", err)
54
-
}
55
-
56
-
summaries := lop.Map(keys, func(key string, _ int) *StoreSummary {
57
-
reader, err := c.Get(ctx, StoreCachePrefix+key)
58
-
if err != nil {
59
-
slog.WarnContext(ctx, "failed to read cached heb store summary", "key", key, "error", err)
60
-
return nil
61
-
}
62
-
defer func() {
63
-
_ = reader.Close()
64
-
}()
65
-
66
-
var summary StoreSummary
67
-
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
68
-
slog.WarnContext(ctx, "failed to decode cached heb store summary", "key", key, "error", err)
69
-
return nil
49
+
func RebuildLocationIndex(ctx context.Context, c cache.ListCache, zipLookup storeindex.ZipCentroidLookup) error {
50
+
_, err := storeindex.RebuildFromStoreSummaries(ctx, c, StoreCachePrefix, LocationIndexCacheKey, func(summary StoreSummary) storeindex.Entry {
51
+
lat, lon := storeindex.Coordinates(summary.Lat, summary.Lon, summary.ZipCode, zipLookup)
52
+
return storeindex.Entry{
53
+
ID: summary.ID,
54
+
Lat: lat,
55
+
Lon: lon,
70
56
}
71
-
return &summary
72
57
})
58
+
return err
59
+
}
73
60
74
-
summaries = lo.Compact(summaries)
75
-
if len(summaries) == 0 {
76
-
return nil, fmt.Errorf("failed to load heb locations")
61
+
type loader struct {
62
+
cache cache.Cache
63
+
}
64
+
65
+
func (l *loader) Load(ctx context.Context, locationID string) (locationtypes.Location, error) {
66
+
reader, err := l.cache.Get(ctx, StoreCachePrefix+locationID)
67
+
if err != nil {
68
+
return locationtypes.Location{}, err
77
69
}
78
-
slog.InfoContext(ctx, "loaded heb locations", "count", len(summaries))
70
+
defer func() {
71
+
_ = reader.Close()
72
+
}()
79
73
80
-
return summaries, nil
81
-
}
82
-
83
-
func storeSummaryToLocation(summary StoreSummary) locationtypes.Location {
74
+
var summary StoreSummary
75
+
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
76
+
return locationtypes.Location{}, fmt.Errorf("decode heb store summary: %w", err)
77
+
}
84
78
return locationtypes.Location{
85
79
ID: summary.ID,
86
80
Name: summary.Name,
···
90
84
Lat: summary.Lat,
91
85
Lon: summary.Lon,
92
86
Chain: Container,
93
-
}
87
+
}, nil
94
88
}
+20
-20
internal/heb/locations.go
+20
-20
internal/heb/locations.go
···
7
7
8
8
"careme/internal/cache"
9
9
"careme/internal/config"
10
+
"careme/internal/locations/hydrator"
10
11
"careme/internal/locations/nearby"
12
+
"careme/internal/locations/storeindex"
13
+
11
14
locationtypes "careme/internal/locations/types"
12
15
)
13
16
···
17
20
18
21
type LocationBackend struct {
19
22
zipLookup centroidByZip
20
-
byID map[string]locationtypes.Location
23
+
spatial []locationtypes.Location
24
+
hydrator *hydrator.LazyHydrator
21
25
}
22
26
23
27
func NewLocationBackendFromConfig(ctx context.Context, cfg *config.Config, zipLookup centroidByZip) (*LocationBackend, error) {
···
37
41
return newLocationBackend(ctx, listCache, zipLookup)
38
42
}
39
43
40
-
func newLocationBackend(ctx context.Context, c cache.ListCache, zipLookup centroidByZip) (*LocationBackend, error) {
44
+
func newLocationBackend(ctx context.Context, c cache.Cache, zipLookup centroidByZip) (*LocationBackend, error) {
41
45
if c == nil {
42
-
return nil, fmt.Errorf("list cache is required")
46
+
return nil, fmt.Errorf("cache is required")
43
47
}
44
48
45
-
summaries, err := loadCachedStoreSummaries(ctx, c)
49
+
entries, err := storeindex.Load(ctx, c, LocationIndexCacheKey)
46
50
if err != nil {
47
-
return nil, err
51
+
return nil, fmt.Errorf("load heb locations index: %w", err)
48
52
}
49
53
50
-
byID := make(map[string]locationtypes.Location, len(summaries))
51
-
for _, summary := range summaries {
52
-
loc := storeSummaryToLocation(*summary)
53
-
byID[loc.ID] = loc
54
+
spatial := make([]locationtypes.Location, 0, len(entries))
55
+
for _, entry := range entries {
56
+
spatial = append(spatial, entry.ToLocation())
54
57
}
55
58
56
59
return &LocationBackend{
57
60
zipLookup: zipLookup,
58
-
byID: byID,
61
+
spatial: spatial,
62
+
hydrator: hydrator.NewLazyHydrator(&loader{c}),
59
63
}, nil
60
64
}
61
65
···
67
71
return false
68
72
}
69
73
70
-
func (b *LocationBackend) GetLocationByID(_ context.Context, locationID string) (*locationtypes.Location, error) {
74
+
func (b *LocationBackend) GetLocationByID(ctx context.Context, locationID string) (*locationtypes.Location, error) {
71
75
locationID = strings.TrimSpace(locationID)
72
76
if !IsID(locationID) {
73
77
return nil, fmt.Errorf("heb location id %q is invalid", locationID)
74
78
}
75
79
76
-
loc, exists := b.byID[locationID]
77
-
if !exists {
78
-
return nil, fmt.Errorf("heb location %q not found", locationID)
80
+
loc, err := b.hydrator.Hydrate(ctx, locationID)
81
+
if err != nil {
82
+
return nil, err
79
83
}
80
-
81
84
copy := loc
82
85
return ©, nil
83
86
}
84
87
85
88
func (b *LocationBackend) GetLocationsByZip(ctx context.Context, zipcode string) ([]locationtypes.Location, error) {
86
-
candidates := make([]locationtypes.Location, 0, len(b.byID))
87
-
for _, loc := range b.byID {
88
-
candidates = append(candidates, loc)
89
-
}
90
-
return nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, candidates, nearby.MaxLocationDistanceMiles), nil
89
+
candidates := nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, b.spatial, nearby.MaxLocationDistanceMiles)
90
+
return storeindex.HydrateLocations(ctx, candidates, b.hydrator.Hydrate)
91
91
}
+21
-8
internal/heb/locations_test.go
+21
-8
internal/heb/locations_test.go
···
16
16
if err := CacheStoreSummary(context.Background(), cacheStore, robstownSummary()); err != nil {
17
17
t.Fatalf("CacheStoreSummary returned error: %v", err)
18
18
}
19
-
20
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
19
+
zipLookup := staticZIPLookup{
21
20
"78380": {Lat: 27.8000, Lon: -97.6700},
22
-
})
21
+
}
22
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
23
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
24
+
}
25
+
26
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
23
27
if err != nil {
24
28
t.Fatalf("newLocationBackend returned error: %v", err)
25
29
}
···
35
39
if loc.Name != "Robstown H-E-B" || loc.ZipCode != "78380" || loc.Chain != "heb" {
36
40
t.Fatalf("unexpected location: %+v", loc)
37
41
}
42
+
reader, err := cacheStore.Get(context.Background(), LocationIndexCacheKey)
43
+
if err != nil {
44
+
t.Fatalf("expected compact location index to be cached: %v", err)
45
+
}
46
+
_ = reader.Close()
38
47
}
39
48
40
49
func TestLocationBackendGetLocationsByZipUsesDistance(t *testing.T) {
···
47
56
if err := CacheStoreSummary(context.Background(), cacheStore, farStoreSummary()); err != nil {
48
57
t.Fatalf("cache far store summary: %v", err)
49
58
}
50
-
51
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
59
+
zipLookup := staticZIPLookup{
52
60
"78380": {Lat: 27.8000, Lon: -97.6700},
53
-
})
61
+
}
62
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
63
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
64
+
}
65
+
66
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
54
67
if err != nil {
55
68
t.Fatalf("newLocationBackend returned error: %v", err)
56
69
}
···
76
89
if err == nil {
77
90
t.Fatal("expected newLocationBackend to return an error")
78
91
}
79
-
if !strings.Contains(err.Error(), "failed to load heb locations") {
80
-
t.Fatalf("expected missing summaries error, got %v", err)
92
+
if !strings.Contains(err.Error(), "load heb locations index") {
93
+
t.Fatalf("expected missing index error, got %v", err)
81
94
}
82
95
}
83
96
+3
internal/locations/albertsons_test.go
+3
internal/locations/albertsons_test.go
···
48
48
}); err != nil {
49
49
t.Fatalf("CacheStoreSummary returned error: %v", err)
50
50
}
51
+
if err := albertsons.RebuildLocationIndex(context.Background(), listCache, LoadCentroids()); err != nil {
52
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
53
+
}
51
54
52
55
storage, err := New(&config.Config{
53
56
Albertsons: config.AlbertsonsConfig{Enable: true},
+3
internal/locations/aldi_test.go
+3
internal/locations/aldi_test.go
···
48
48
}); err != nil {
49
49
t.Fatalf("CacheStoreSummary returned error: %v", err)
50
50
}
51
+
if err := aldi.RebuildLocationIndex(context.Background(), listCache, LoadCentroids()); err != nil {
52
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
53
+
}
51
54
52
55
storage, err := New(&config.Config{
53
56
Aldi: config.AldiConfig{Enable: true},
+3
internal/locations/heb_test.go
+3
internal/locations/heb_test.go
···
47
47
}); err != nil {
48
48
t.Fatalf("CacheStoreSummary returned error: %v", err)
49
49
}
50
+
if err := heb.RebuildLocationIndex(context.Background(), listCache, LoadCentroids()); err != nil {
51
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
52
+
}
50
53
51
54
storage, err := New(&config.Config{
52
55
HEB: config.HEBConfig{Enable: true},
+46
internal/locations/hydrator/hydrator.go
+46
internal/locations/hydrator/hydrator.go
···
1
+
package hydrator
2
+
3
+
import (
4
+
"context"
5
+
"sync"
6
+
7
+
locationtypes "careme/internal/locations/types"
8
+
)
9
+
10
+
// could take subset of cache
11
+
12
+
type loader interface {
13
+
Load(context.Context, string) (locationtypes.Location, error)
14
+
}
15
+
16
+
type LazyHydrator struct {
17
+
loader loader
18
+
mu sync.RWMutex
19
+
byID map[string]locationtypes.Location
20
+
}
21
+
22
+
func NewLazyHydrator(loader loader) *LazyHydrator {
23
+
return &LazyHydrator{
24
+
loader: loader,
25
+
byID: make(map[string]locationtypes.Location),
26
+
}
27
+
}
28
+
29
+
func (h *LazyHydrator) Hydrate(ctx context.Context, locationID string) (locationtypes.Location, error) {
30
+
h.mu.RLock()
31
+
loc, ok := h.byID[locationID]
32
+
h.mu.RUnlock()
33
+
if ok {
34
+
return loc, nil
35
+
}
36
+
37
+
loc, err := h.loader.Load(ctx, locationID)
38
+
if err != nil {
39
+
return locationtypes.Location{}, err
40
+
}
41
+
42
+
h.mu.Lock()
43
+
h.byID[locationID] = loc
44
+
h.mu.Unlock()
45
+
return loc, nil
46
+
}
+61
internal/locations/hydrator/hydrator_test.go
+61
internal/locations/hydrator/hydrator_test.go
···
1
+
package hydrator
2
+
3
+
import (
4
+
"context"
5
+
"errors"
6
+
"testing"
7
+
8
+
locationtypes "careme/internal/locations/types"
9
+
)
10
+
11
+
type fakeLoader struct {
12
+
load func(context.Context, string) (locationtypes.Location, error)
13
+
}
14
+
15
+
func (f fakeLoader) Load(ctx context.Context, locationID string) (locationtypes.Location, error) {
16
+
return f.load(ctx, locationID)
17
+
}
18
+
19
+
func TestLazyHydratorCachesLoadedLocations(t *testing.T) {
20
+
t.Parallel()
21
+
22
+
loads := 0
23
+
hydrator := NewLazyHydrator(fakeLoader{
24
+
load: func(_ context.Context, locationID string) (locationtypes.Location, error) {
25
+
loads++
26
+
return locationtypes.Location{ID: locationID, Name: "Loaded"}, nil
27
+
},
28
+
})
29
+
30
+
first, err := hydrator.Hydrate(context.Background(), "store-1")
31
+
if err != nil {
32
+
t.Fatalf("first hydrate: %v", err)
33
+
}
34
+
second, err := hydrator.Hydrate(context.Background(), "store-1")
35
+
if err != nil {
36
+
t.Fatalf("second hydrate: %v", err)
37
+
}
38
+
39
+
if loads != 1 {
40
+
t.Fatalf("expected one load, got %d", loads)
41
+
}
42
+
if first != second {
43
+
t.Fatalf("expected cached location on second hydrate")
44
+
}
45
+
}
46
+
47
+
func TestLazyHydratorReturnsLoaderError(t *testing.T) {
48
+
t.Parallel()
49
+
50
+
wantErr := errors.New("boom")
51
+
hydrator := NewLazyHydrator(fakeLoader{
52
+
load: func(_ context.Context, _ string) (locationtypes.Location, error) {
53
+
return locationtypes.Location{}, wantErr
54
+
},
55
+
})
56
+
57
+
_, err := hydrator.Hydrate(context.Background(), "store-1")
58
+
if !errors.Is(err, wantErr) {
59
+
t.Fatalf("expected %v, got %v", wantErr, err)
60
+
}
61
+
}
+130
internal/locations/storeindex/storeindex.go
+130
internal/locations/storeindex/storeindex.go
···
1
+
package storeindex
2
+
3
+
import (
4
+
"bytes"
5
+
"context"
6
+
"encoding/json"
7
+
"fmt"
8
+
"log/slog"
9
+
10
+
"careme/internal/cache"
11
+
"careme/internal/parallelism"
12
+
13
+
locationtypes "careme/internal/locations/types"
14
+
)
15
+
16
+
// TODO should we just embed zip centroid?
17
+
type Entry struct {
18
+
ID string `json:"id"`
19
+
Lat *float64 `json:"lat,omitempty"`
20
+
Lon *float64 `json:"lon,omitempty"`
21
+
}
22
+
23
+
type ZipCentroidLookup interface {
24
+
ZipCentroidByZIP(zip string) (locationtypes.ZipCentroid, bool)
25
+
}
26
+
27
+
func (e Entry) ToLocation() locationtypes.Location {
28
+
return locationtypes.Location{
29
+
ID: e.ID,
30
+
Lat: e.Lat,
31
+
Lon: e.Lon,
32
+
}
33
+
}
34
+
35
+
func Coordinates(lat, lon *float64, zipCode string, zipLookup ZipCentroidLookup) (*float64, *float64) {
36
+
if lat != nil && lon != nil {
37
+
return lat, lon
38
+
}
39
+
40
+
centroid, ok := zipLookup.ZipCentroidByZIP(zipCode)
41
+
if !ok {
42
+
// panic("missing zip centroid for zip code: " + zipCode)
43
+
return nil, nil // boo what will location.go do with this?
44
+
}
45
+
46
+
return ¢roid.Lat, ¢roid.Lon
47
+
}
48
+
49
+
func HydrateLocations(ctx context.Context, candidates []locationtypes.Location, hydrate func(context.Context, string) (locationtypes.Location, error)) ([]locationtypes.Location, error) {
50
+
if len(candidates) == 0 {
51
+
return nil, nil
52
+
}
53
+
54
+
out, err := parallelism.MapWithErrors(candidates, func(candidate locationtypes.Location) (locationtypes.Location, error) {
55
+
return hydrate(ctx, candidate.ID)
56
+
})
57
+
58
+
if len(out) == 0 {
59
+
return nil, fmt.Errorf("zero hydrated locations: %w", err)
60
+
}
61
+
return out, nil
62
+
}
63
+
64
+
func Save(ctx context.Context, c cache.Cache, key string, entries []Entry) error {
65
+
raw, err := json.Marshal(entries)
66
+
if err != nil {
67
+
return fmt.Errorf("marshal location index: %w", err)
68
+
}
69
+
if err := c.PutReader(ctx, key, bytes.NewReader(raw), cache.Unconditional()); err != nil {
70
+
return fmt.Errorf("write location index: %w", err)
71
+
}
72
+
return nil
73
+
}
74
+
75
+
func Load(ctx context.Context, c cache.Cache, key string) ([]Entry, error) {
76
+
reader, err := c.Get(ctx, key)
77
+
if err != nil {
78
+
return nil, err
79
+
}
80
+
defer func() {
81
+
_ = reader.Close()
82
+
}()
83
+
84
+
var entries []Entry
85
+
if err := json.NewDecoder(reader).Decode(&entries); err != nil {
86
+
return nil, fmt.Errorf("decode index: %w", err)
87
+
}
88
+
if len(entries) == 0 {
89
+
return nil, fmt.Errorf("zero entry index")
90
+
}
91
+
return entries, nil
92
+
}
93
+
94
+
func RebuildFromStoreSummaries[T any](ctx context.Context, c cache.ListCache, storePrefix, indexKey string, toEntry func(T) Entry) ([]Entry, error) {
95
+
keys, err := c.List(ctx, storePrefix, "")
96
+
if err != nil {
97
+
return nil, fmt.Errorf("list cached store summaries: %w", err)
98
+
}
99
+
100
+
// could parallize but less important now that its in scrapers
101
+
entries := make([]Entry, 0, len(keys))
102
+
for _, key := range keys {
103
+
reader, err := c.Get(ctx, storePrefix+key)
104
+
if err != nil {
105
+
return nil, fmt.Errorf("read cached store summary: %w", err)
106
+
}
107
+
defer func() {
108
+
_ = reader.Close()
109
+
}()
110
+
111
+
var summary T
112
+
decodeErr := json.NewDecoder(reader).Decode(&summary)
113
+
if decodeErr != nil {
114
+
return nil, fmt.Errorf("decode cached store summary: %w", decodeErr)
115
+
}
116
+
117
+
entries = append(entries, toEntry(summary))
118
+
}
119
+
120
+
if len(entries) == 0 {
121
+
return nil, fmt.Errorf("zero location index")
122
+
}
123
+
124
+
if err := Save(ctx, c, indexKey, entries); err != nil {
125
+
return nil, err
126
+
}
127
+
128
+
slog.InfoContext(ctx, "rebuilt compact location index", "count", len(entries))
129
+
return entries, nil
130
+
}
+3
internal/locations/wholefoods_test.go
+3
internal/locations/wholefoods_test.go
···
46
46
}); err != nil {
47
47
t.Fatalf("CacheStoreSummary returned error: %v", err)
48
48
}
49
+
if err := wholefoods.RebuildLocationIndex(context.Background(), listCache, LoadCentroids()); err != nil {
50
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
51
+
}
49
52
50
53
storage, err := New(&config.Config{
51
54
WholeFoods: config.WholeFoodsConfig{Enable: true},
+29
internal/parallelism/flatten.go
+29
internal/parallelism/flatten.go
···
34
34
35
35
return merged, errors.Join(errs...)
36
36
}
37
+
38
+
// MapWithErrors collects errors but doesn't cancel anything.
39
+
func MapWithErrors[T any, T2 any](items []T, fn func(T) (T2, error)) ([]T2, error) {
40
+
if len(items) == 0 {
41
+
return []T2{}, nil
42
+
}
43
+
44
+
type result struct {
45
+
value T2
46
+
err error
47
+
}
48
+
49
+
mapped := lop.Map(items, func(item T, _ int) result {
50
+
value, err := fn(item)
51
+
return result{value: value, err: err}
52
+
})
53
+
54
+
merged := make([]T2, 0)
55
+
errs := make([]error, 0)
56
+
for _, r := range mapped {
57
+
if r.err != nil {
58
+
errs = append(errs, r.err)
59
+
continue
60
+
}
61
+
merged = append(merged, r.value)
62
+
}
63
+
64
+
return merged, errors.Join(errs...)
65
+
}
+93
internal/parallelism/flatten_test.go
+93
internal/parallelism/flatten_test.go
···
3
3
import (
4
4
"errors"
5
5
"slices"
6
+
"sync/atomic"
6
7
"testing"
8
+
"time"
7
9
)
8
10
9
11
func TestFlatten_MergesResultsAndErrors(t *testing.T) {
···
49
51
t.Fatalf("expected empty result for empty input, got: %v", got)
50
52
}
51
53
}
54
+
55
+
func TestMapWithErrors_ReturnsSuccessfulValuesInInputOrderAndJoinsErrors(t *testing.T) {
56
+
errOne := errors.New("err one")
57
+
errTwo := errors.New("err two")
58
+
59
+
got, err := MapWithErrors([]int{1, 2, 3, 4}, func(i int) (string, error) {
60
+
switch i {
61
+
case 1:
62
+
return "a", nil
63
+
case 2:
64
+
return "b", errOne
65
+
case 3:
66
+
return "", errTwo
67
+
case 4:
68
+
return "d", nil
69
+
default:
70
+
return "", nil
71
+
}
72
+
})
73
+
74
+
want := []string{"a", "d"}
75
+
if !slices.Equal(got, want) {
76
+
t.Fatalf("unexpected mapped results: got=%v want=%v", got, want)
77
+
}
78
+
if !errors.Is(err, errOne) {
79
+
t.Fatalf("expected merged error to include errOne, got: %v", err)
80
+
}
81
+
if !errors.Is(err, errTwo) {
82
+
t.Fatalf("expected merged error to include errTwo, got: %v", err)
83
+
}
84
+
}
85
+
86
+
func TestMapWithErrors_DoesNotCancelRemainingWorkAfterError(t *testing.T) {
87
+
errBoom := errors.New("boom")
88
+
started := make(chan int, 3)
89
+
release := make(chan struct{})
90
+
done := make(chan struct{})
91
+
var calls atomic.Int32
92
+
93
+
var got []int
94
+
var gotErr error
95
+
go func() {
96
+
defer close(done)
97
+
got, gotErr = MapWithErrors([]int{1, 2, 3}, func(i int) (int, error) {
98
+
calls.Add(1)
99
+
started <- i
100
+
if i == 1 {
101
+
return 0, errBoom
102
+
}
103
+
<-release
104
+
return i * 10, nil
105
+
})
106
+
}()
107
+
108
+
for range 3 {
109
+
select {
110
+
case <-started:
111
+
case <-time.After(2 * time.Second):
112
+
t.Fatal("timed out waiting for all workers to start")
113
+
}
114
+
}
115
+
close(release)
116
+
117
+
select {
118
+
case <-done:
119
+
case <-time.After(2 * time.Second):
120
+
t.Fatal("timed out waiting for MapWithErrors to finish")
121
+
}
122
+
123
+
if calls.Load() != 3 {
124
+
t.Fatalf("expected all items to run, got %d calls", calls.Load())
125
+
}
126
+
if !slices.Equal(got, []int{20, 30}) {
127
+
t.Fatalf("unexpected mapped results: got=%v want=%v", got, []int{20, 30})
128
+
}
129
+
if !errors.Is(gotErr, errBoom) {
130
+
t.Fatalf("expected merged error to include errBoom, got: %v", gotErr)
131
+
}
132
+
}
133
+
134
+
func TestMapWithErrors_EmptyInput(t *testing.T) {
135
+
got, err := MapWithErrors([]string{}, func(s string) (int, error) {
136
+
return 1, nil
137
+
})
138
+
if err != nil {
139
+
t.Fatalf("expected nil error for empty input, got: %v", err)
140
+
}
141
+
if len(got) != 0 {
142
+
t.Fatalf("expected empty result for empty input, got: %v", got)
143
+
}
144
+
}
+35
-33
internal/publix/cache.go
+35
-33
internal/publix/cache.go
···
5
5
"encoding/json"
6
6
"errors"
7
7
"fmt"
8
-
"log/slog"
9
8
"slices"
9
+
"strings"
10
10
11
11
"careme/internal/cache"
12
+
"careme/internal/locations/storeindex"
13
+
12
14
locationtypes "careme/internal/locations/types"
13
-
14
-
"github.com/samber/lo"
15
-
lop "github.com/samber/lo/parallel"
16
15
)
17
16
18
17
const (
···
21
20
StoreURLMapCacheKey = "publix/store_url_map.json"
22
21
MissingStoreIDsCacheKey = "publix/missing_store_ids.json"
23
22
LocationIDPrefix = "publix_"
23
+
LocationIndexCacheKey = "publix/store_locations.json"
24
24
)
25
25
26
26
func SaveMissingStoreIDs(ctx context.Context, c cache.Cache, ids map[string]struct{}) error {
···
86
86
return nil
87
87
}
88
88
89
-
func loadCachedStoreSummaries(ctx context.Context, c cache.ListCache) ([]*StoreSummary, error) {
90
-
keys, err := c.List(ctx, StoreCachePrefix, "")
91
-
if err != nil {
92
-
return nil, fmt.Errorf("list cached store summaries: %w", err)
93
-
}
94
-
95
-
summaries := lop.Map(keys, func(key string, _ int) *StoreSummary {
96
-
reader, err := c.Get(ctx, StoreCachePrefix+key)
97
-
if err != nil {
98
-
slog.WarnContext(ctx, "failed to read cached publix store summary", "key", key, "error", err)
99
-
return nil
100
-
}
101
-
defer func() {
102
-
_ = reader.Close()
103
-
}()
89
+
func RebuildLocationIndex(ctx context.Context, c cache.ListCache, zipLookup storeindex.ZipCentroidLookup) error {
90
+
_, err := storeindex.RebuildFromStoreSummaries(ctx, c, StoreCachePrefix, LocationIndexCacheKey,
91
+
func(summary StoreSummary) storeindex.Entry {
92
+
lat, lon := storeindex.Coordinates(summary.Lat, summary.Lon, summary.ZipCode, zipLookup)
93
+
return storeindex.Entry{
94
+
ID: summary.ID,
95
+
Lat: lat,
96
+
Lon: lon,
97
+
}
98
+
})
99
+
return err
100
+
}
104
101
105
-
var summary StoreSummary
106
-
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
107
-
slog.WarnContext(ctx, "failed to decode cached publix store summary", "key", key, "error", err)
108
-
return nil
109
-
}
110
-
return &summary
111
-
})
102
+
type loader struct {
103
+
cache cache.Cache
104
+
}
112
105
113
-
summaries = lo.Compact(summaries)
114
-
if len(summaries) == 0 {
115
-
return nil, fmt.Errorf("failed to load publix locations")
106
+
func (l *loader) Load(ctx context.Context, locationID string) (locationtypes.Location, error) {
107
+
storeID := strings.TrimPrefix(strings.TrimSpace(locationID), LocationIDPrefix)
108
+
if storeID == "" {
109
+
return locationtypes.Location{}, fmt.Errorf("publix location %q not found", locationID)
116
110
}
117
111
118
-
return summaries, nil
119
-
}
112
+
reader, err := l.cache.Get(ctx, StoreCachePrefix+storeID)
113
+
if err != nil {
114
+
return locationtypes.Location{}, err
115
+
}
116
+
defer func() {
117
+
_ = reader.Close()
118
+
}()
120
119
121
-
func storeSummaryToLocation(summary StoreSummary) locationtypes.Location {
120
+
var summary StoreSummary
121
+
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
122
+
return locationtypes.Location{}, fmt.Errorf("decode publix store summary: %w", err)
123
+
}
122
124
return locationtypes.Location{
123
125
ID: summary.ID,
124
126
Name: summary.Name,
···
128
130
Lat: summary.Lat,
129
131
Lon: summary.Lon,
130
132
Chain: "publix",
131
-
}
133
+
}, nil
132
134
}
+19
-19
internal/publix/locations.go
+19
-19
internal/publix/locations.go
···
7
7
8
8
"careme/internal/cache"
9
9
"careme/internal/config"
10
+
"careme/internal/locations/hydrator"
10
11
"careme/internal/locations/nearby"
12
+
"careme/internal/locations/storeindex"
13
+
11
14
locationtypes "careme/internal/locations/types"
12
15
)
13
16
···
17
20
18
21
type LocationBackend struct {
19
22
zipLookup centroidByZip
20
-
byID map[string]locationtypes.Location
23
+
spatial []locationtypes.Location
24
+
hydrator *hydrator.LazyHydrator
21
25
}
22
26
23
27
func NewLocationBackendFromConfig(ctx context.Context, cfg *config.Config, zipLookup centroidByZip) (*LocationBackend, error) {
···
37
41
return newLocationBackend(ctx, listCache, zipLookup)
38
42
}
39
43
40
-
func newLocationBackend(ctx context.Context, c cache.ListCache, zipLookup centroidByZip) (*LocationBackend, error) {
41
-
summaries, err := loadCachedStoreSummaries(ctx, c)
44
+
func newLocationBackend(ctx context.Context, c cache.Cache, zipLookup centroidByZip) (*LocationBackend, error) {
45
+
entries, err := storeindex.Load(ctx, c, LocationIndexCacheKey)
42
46
if err != nil {
43
-
return nil, err
47
+
return nil, fmt.Errorf("load publix locations index: %w", err)
44
48
}
45
49
46
-
byID := make(map[string]locationtypes.Location, len(summaries))
47
-
for _, summary := range summaries {
48
-
loc := storeSummaryToLocation(*summary)
49
-
byID[loc.ID] = loc
50
+
spatial := make([]locationtypes.Location, 0, len(entries))
51
+
for _, entry := range entries {
52
+
spatial = append(spatial, entry.ToLocation())
50
53
}
51
54
52
55
return &LocationBackend{
53
56
zipLookup: zipLookup,
54
-
byID: byID,
57
+
spatial: spatial,
58
+
hydrator: hydrator.NewLazyHydrator(&loader{c}),
55
59
}, nil
56
60
}
57
61
···
64
68
return false
65
69
}
66
70
67
-
func (b *LocationBackend) GetLocationByID(_ context.Context, locationID string) (*locationtypes.Location, error) {
71
+
func (b *LocationBackend) GetLocationByID(ctx context.Context, locationID string) (*locationtypes.Location, error) {
68
72
locationID = strings.TrimSpace(locationID)
69
73
if !b.IsID(locationID) {
70
74
return nil, fmt.Errorf("publix location id %q is invalid", locationID)
71
75
}
72
76
73
-
loc, exists := b.byID[locationID]
74
-
if !exists {
75
-
return nil, fmt.Errorf("publix location %q not found", locationID)
77
+
loc, err := b.hydrator.Hydrate(ctx, locationID)
78
+
if err != nil {
79
+
return nil, err
76
80
}
77
-
78
81
copy := loc
79
82
return ©, nil
80
83
}
81
84
82
85
func (b *LocationBackend) GetLocationsByZip(ctx context.Context, zipcode string) ([]locationtypes.Location, error) {
83
-
candidates := make([]locationtypes.Location, 0, len(b.byID))
84
-
for _, loc := range b.byID {
85
-
candidates = append(candidates, loc)
86
-
}
87
-
return nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, candidates, nearby.MaxLocationDistanceMiles), nil
86
+
candidates := nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, b.spatial, nearby.MaxLocationDistanceMiles)
87
+
return storeindex.HydrateLocations(ctx, candidates, b.hydrator.Hydrate)
88
88
}
+21
-8
internal/publix/locations_test.go
+21
-8
internal/publix/locations_test.go
···
16
16
if err := CacheStoreSummary(context.Background(), cacheStore, nearbySummary()); err != nil {
17
17
t.Fatalf("CacheStoreSummary returned error: %v", err)
18
18
}
19
-
20
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
19
+
zipLookup := staticZIPLookup{
21
20
"35401": {Lat: 33.2091, Lon: -87.5692},
22
-
})
21
+
}
22
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
23
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
24
+
}
25
+
26
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
23
27
if err != nil {
24
28
t.Fatalf("newLocationBackend returned error: %v", err)
25
29
}
···
35
39
if loc.Name != "Publix at University Town Center" || loc.ZipCode != "35401" || loc.Chain != "publix" {
36
40
t.Fatalf("unexpected location: %+v", loc)
37
41
}
42
+
reader, err := cacheStore.Get(context.Background(), LocationIndexCacheKey)
43
+
if err != nil {
44
+
t.Fatalf("expected compact location index to be cached: %v", err)
45
+
}
46
+
_ = reader.Close()
38
47
}
39
48
40
49
func TestLocationBackendGetLocationsByZipUsesDistance(t *testing.T) {
···
47
56
if err := CacheStoreSummary(context.Background(), cacheStore, farSummary()); err != nil {
48
57
t.Fatalf("cache far summary: %v", err)
49
58
}
50
-
51
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
59
+
zipLookup := staticZIPLookup{
52
60
"35401": {Lat: 33.2091, Lon: -87.5692},
53
-
})
61
+
}
62
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
63
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
64
+
}
65
+
66
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
54
67
if err != nil {
55
68
t.Fatalf("newLocationBackend returned error: %v", err)
56
69
}
···
76
89
if err == nil {
77
90
t.Fatal("expected newLocationBackend to return an error")
78
91
}
79
-
if !strings.Contains(err.Error(), "failed to load publix locations") {
80
-
t.Fatalf("expected missing summaries error, got %v", err)
92
+
if !strings.Contains(err.Error(), "load publix locations index") {
93
+
t.Fatalf("expected missing index error, got %v", err)
81
94
}
82
95
}
83
96
+36
-33
internal/wegmans/cache.go
+36
-33
internal/wegmans/cache.go
···
5
5
"encoding/json"
6
6
"errors"
7
7
"fmt"
8
-
"log/slog"
9
8
"strconv"
9
+
"strings"
10
10
11
11
"careme/internal/cache"
12
+
"careme/internal/locations/storeindex"
13
+
12
14
locationtypes "careme/internal/locations/types"
15
+
)
13
16
14
-
"github.com/samber/lo"
15
-
lop "github.com/samber/lo/parallel"
16
-
)
17
+
const LocationIndexCacheKey = "wegmans/store_locations.json"
17
18
18
19
func CacheStoreSummary(ctx context.Context, c cache.Cache, summary *StoreSummary) error {
19
20
if summary == nil {
···
34
35
return nil
35
36
}
36
37
37
-
func loadCachedStoreSummaries(ctx context.Context, c cache.ListCache) ([]*StoreSummary, error) {
38
-
keys, err := c.List(ctx, StoreCachePrefix, "")
39
-
if err != nil {
40
-
return nil, fmt.Errorf("list cached store summaries: %w", err)
41
-
}
38
+
func RebuildLocationIndex(ctx context.Context, c cache.ListCache, zipLookup storeindex.ZipCentroidLookup) error {
39
+
_, err := storeindex.RebuildFromStoreSummaries[StoreSummary](ctx, c, StoreCachePrefix, LocationIndexCacheKey,
40
+
func(summary StoreSummary) storeindex.Entry {
41
+
lat, lon := storeindex.Coordinates(summary.Lat, summary.Lon, summary.ZipCode, zipLookup)
42
+
return storeindex.Entry{
43
+
ID: summary.ID,
44
+
Lat: lat,
45
+
Lon: lon,
46
+
}
47
+
})
48
+
return err
49
+
}
42
50
43
-
summaries := lop.Map(keys, func(key string, _ int) *StoreSummary {
44
-
reader, err := c.Get(ctx, StoreCachePrefix+key)
45
-
if err != nil {
46
-
slog.WarnContext(ctx, "failed to read cached wegmans store summary", "key", key, "error", err)
47
-
return nil
48
-
}
49
-
defer func() {
50
-
_ = reader.Close()
51
-
}()
51
+
type loader struct {
52
+
cache cache.Cache
53
+
}
52
54
53
-
var summary StoreSummary
54
-
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
55
-
slog.WarnContext(ctx, "failed to decode cached wegmans store summary", "key", key, "error", err)
56
-
return nil
57
-
}
58
-
return &summary
59
-
})
60
-
61
-
summaries = lo.Compact(summaries)
62
-
if len(summaries) == 0 {
63
-
return nil, fmt.Errorf("failed to load wegmans locations")
55
+
func (l *loader) Load(ctx context.Context, locationID string) (locationtypes.Location, error) {
56
+
storeNumber := strings.TrimPrefix(strings.TrimSpace(locationID), LocationIDPrefix)
57
+
if storeNumber == "" {
58
+
return locationtypes.Location{}, fmt.Errorf("wegmans location %q not found", locationID)
64
59
}
65
60
66
-
return summaries, nil
67
-
}
61
+
reader, err := l.cache.Get(ctx, StoreCachePrefix+storeNumber)
62
+
if err != nil {
63
+
return locationtypes.Location{}, err
64
+
}
65
+
defer func() {
66
+
_ = reader.Close()
67
+
}()
68
68
69
-
func StoreSummaryToLocation(summary StoreSummary) locationtypes.Location {
69
+
var summary StoreSummary
70
+
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
71
+
return locationtypes.Location{}, fmt.Errorf("decode wegmans store summary: %w", err)
72
+
}
70
73
return locationtypes.Location{
71
74
ID: summary.ID,
72
75
Name: summary.Name,
···
76
79
Lat: summary.Lat,
77
80
Lon: summary.Lon,
78
81
Chain: Container,
79
-
}
82
+
}, nil
80
83
}
+18
-19
internal/wegmans/locations.go
+18
-19
internal/wegmans/locations.go
···
7
7
8
8
"careme/internal/cache"
9
9
"careme/internal/config"
10
+
"careme/internal/locations/hydrator"
10
11
"careme/internal/locations/nearby"
12
+
"careme/internal/locations/storeindex"
11
13
12
14
locationtypes "careme/internal/locations/types"
13
15
)
···
18
20
19
21
type LocationBackend struct {
20
22
zipLookup centroidByZip
21
-
byID map[string]locationtypes.Location
23
+
spatial []locationtypes.Location
24
+
hydrator *hydrator.LazyHydrator
22
25
}
23
26
24
27
func NewLocationBackend(ctx context.Context, cfg *config.Config, zipLookup centroidByZip) (*LocationBackend, error) {
···
39
42
return newLocationBackend(ctx, listCache, zipLookup)
40
43
}
41
44
42
-
func newLocationBackend(ctx context.Context, c cache.ListCache, zipLookup centroidByZip) (*LocationBackend, error) {
43
-
summaries, err := loadCachedStoreSummaries(ctx, c)
45
+
func newLocationBackend(ctx context.Context, c cache.Cache, zipLookup centroidByZip) (*LocationBackend, error) {
46
+
entries, err := storeindex.Load(ctx, c, LocationIndexCacheKey)
44
47
if err != nil {
45
-
return nil, err
48
+
return nil, fmt.Errorf("load wegmans locations index: %w", err)
46
49
}
47
50
48
-
byID := make(map[string]locationtypes.Location, len(summaries))
49
-
for _, summary := range summaries {
50
-
loc := StoreSummaryToLocation(*summary)
51
-
byID[loc.ID] = loc
51
+
spatial := make([]locationtypes.Location, 0, len(entries))
52
+
for _, entry := range entries {
53
+
spatial = append(spatial, entry.ToLocation())
52
54
}
53
55
54
56
return &LocationBackend{
55
57
zipLookup: zipLookup,
56
-
byID: byID,
58
+
spatial: spatial,
59
+
hydrator: hydrator.NewLazyHydrator(&loader{c}),
57
60
}, nil
58
61
}
59
62
···
65
68
return false
66
69
}
67
70
68
-
func (b *LocationBackend) GetLocationByID(_ context.Context, locationID string) (*locationtypes.Location, error) {
71
+
func (b *LocationBackend) GetLocationByID(ctx context.Context, locationID string) (*locationtypes.Location, error) {
69
72
locationID = strings.TrimSpace(locationID)
70
73
if !IsID(locationID) {
71
74
return nil, fmt.Errorf("wegmans location id %q is invalid", locationID)
72
75
}
73
76
74
-
loc, exists := b.byID[locationID]
75
-
if !exists {
76
-
return nil, fmt.Errorf("wegmans location %q not found", locationID)
77
+
loc, err := b.hydrator.Hydrate(ctx, locationID)
78
+
if err != nil {
79
+
return nil, err
77
80
}
78
-
79
81
copy := loc
80
82
return ©, nil
81
83
}
82
84
83
85
func (b *LocationBackend) GetLocationsByZip(ctx context.Context, zipcode string) ([]locationtypes.Location, error) {
84
-
candidates := make([]locationtypes.Location, 0, len(b.byID))
85
-
for _, loc := range b.byID {
86
-
candidates = append(candidates, loc)
87
-
}
88
-
return nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, candidates, nearby.MaxLocationDistanceMiles), nil
86
+
candidates := nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, b.spatial, nearby.MaxLocationDistanceMiles)
87
+
return storeindex.HydrateLocations(ctx, candidates, b.hydrator.Hydrate)
89
88
}
90
89
91
90
func IsID(locationID string) bool {
+21
-8
internal/wegmans/locations_test.go
+21
-8
internal/wegmans/locations_test.go
···
17
17
if err := CacheStoreSummary(t.Context(), cacheStore, nearbySummary()); err != nil {
18
18
t.Fatalf("CacheStoreSummary returned error: %v", err)
19
19
}
20
-
21
-
backend, err := newLocationBackend(t.Context(), cacheStore, staticZIPLookup{
20
+
zipLookup := staticZIPLookup{
22
21
"16506": {Lat: 42.0817, Lon: -80.1753},
23
-
})
22
+
}
23
+
if err := RebuildLocationIndex(t.Context(), cacheStore, zipLookup); err != nil {
24
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
25
+
}
26
+
27
+
backend, err := newLocationBackend(t.Context(), cacheStore, zipLookup)
24
28
if err != nil {
25
29
t.Fatalf("NewLocationBackend returned error: %v", err)
26
30
}
···
36
40
if loc.Name != "Wegmans Erie West" || loc.ZipCode != "16506" || loc.Chain != "wegmans" {
37
41
t.Fatalf("unexpected location: %+v", loc)
38
42
}
43
+
reader, err := cacheStore.Get(t.Context(), LocationIndexCacheKey)
44
+
if err != nil {
45
+
t.Fatalf("expected compact location index to be cached: %v", err)
46
+
}
47
+
_ = reader.Close()
39
48
}
40
49
41
50
func TestLocationBackendGetLocationsByZipUsesDistance(t *testing.T) {
···
48
57
if err := CacheStoreSummary(t.Context(), cacheStore, farSummary()); err != nil {
49
58
t.Fatalf("cache far summary: %v", err)
50
59
}
51
-
52
-
backend, err := newLocationBackend(t.Context(), cacheStore, staticZIPLookup{
60
+
zipLookup := staticZIPLookup{
53
61
"16506": {Lat: 42.0817, Lon: -80.1753},
54
-
})
62
+
}
63
+
if err := RebuildLocationIndex(t.Context(), cacheStore, zipLookup); err != nil {
64
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
65
+
}
66
+
67
+
backend, err := newLocationBackend(t.Context(), cacheStore, zipLookup)
55
68
if err != nil {
56
69
t.Fatalf("NewLocationBackend returned error: %v", err)
57
70
}
···
77
90
if err == nil {
78
91
t.Fatal("expected NewLocationBackend to return an error")
79
92
}
80
-
if !strings.Contains(err.Error(), "failed to load wegmans locations") {
81
-
t.Fatalf("expected missing summaries error, got %v", err)
93
+
if !strings.Contains(err.Error(), "load wegmans locations index") {
94
+
t.Fatalf("expected missing index error, got %v", err)
82
95
}
83
96
}
84
97
+40
-34
internal/wholefoods/cache.go
+40
-34
internal/wholefoods/cache.go
···
5
5
"encoding/json"
6
6
"errors"
7
7
"fmt"
8
-
"log/slog"
9
8
"strconv"
9
+
"strings"
10
10
11
11
"careme/internal/cache"
12
-
locationtypes "careme/internal/locations/types"
12
+
"careme/internal/locations/storeindex"
13
13
"careme/internal/sitemapfetch"
14
14
15
-
"github.com/samber/lo"
16
-
lop "github.com/samber/lo/parallel"
15
+
locationtypes "careme/internal/locations/types"
17
16
)
18
17
19
18
const (
···
23
22
StoreURLMapCacheKey = "wholefoods/store_url_map.json"
24
23
LocationIDPrefix = "wholefoods_"
25
24
DefaultStoreSitemapURL = "https://www.wholefoodsmarket.com/sitemap/sitemap-stores.xml"
25
+
LocationIndexCacheKey = "wholefoods/store_locations.json"
26
26
)
27
27
28
28
type StoreReference struct {
···
63
63
return nil
64
64
}
65
65
66
-
// loadCachedStoreSummaries get all store summaries into memory.
67
-
// its pretty intense and maybe we should just load latlong for index
68
-
func loadCachedStoreSummaries(ctx context.Context, c cache.ListCache) ([]*StoreSummaryResponse, error) {
69
-
keys, err := c.List(ctx, StoreCachePrefix, "")
70
-
if err != nil {
71
-
return nil, fmt.Errorf("list cached store summaries: %w", err)
72
-
}
73
-
74
-
summaries := lop.Map(keys, func(key string, i int) *StoreSummaryResponse {
75
-
reader, err := c.Get(ctx, StoreCachePrefix+key)
76
-
if err != nil {
77
-
slog.WarnContext(ctx, "failed to read cached whole foods store summary", "key", key, "error", err)
78
-
return nil
66
+
func RebuildLocationIndex(ctx context.Context, c cache.ListCache, zipLookup storeindex.ZipCentroidLookup) error {
67
+
_, err := storeindex.RebuildFromStoreSummaries(ctx, c, StoreCachePrefix, LocationIndexCacheKey, func(summary StoreSummaryResponse) storeindex.Entry {
68
+
lat, lon := storeindex.Coordinates(&summary.PrimaryLocation.Latitude, &summary.PrimaryLocation.Longitude, summary.PrimaryLocation.Address.ZipCode, zipLookup)
69
+
return storeindex.Entry{
70
+
ID: LocationIDPrefix + strconv.Itoa(summary.StoreID),
71
+
Lat: lat,
72
+
Lon: lon,
79
73
}
80
-
defer func() {
81
-
_ = reader.Close()
82
-
}()
83
-
var summary StoreSummaryResponse
84
-
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
85
-
slog.WarnContext(ctx, "failed to decode cached whole foods store summary", "key", key, "error", err)
86
-
return nil
87
-
}
88
-
return &summary
89
74
})
75
+
return err
76
+
}
90
77
91
-
summaries = lo.Compact(summaries)
78
+
func loadCachedStoreSummaryByID(ctx context.Context, c cache.Cache, locationID string) (*StoreSummaryResponse, error) {
79
+
normalized, ok := parseLocationID(locationID)
80
+
if !ok {
81
+
return nil, fmt.Errorf("whole foods location %q not found", locationID)
82
+
}
83
+
storeID := strings.TrimPrefix(normalized, LocationIDPrefix)
84
+
85
+
reader, err := c.Get(ctx, StoreCachePrefix+storeID)
86
+
if err != nil {
87
+
return nil, err
88
+
}
89
+
defer func() {
90
+
_ = reader.Close()
91
+
}()
92
92
93
-
if len(summaries) == 0 {
94
-
return nil, fmt.Errorf("failed to load wholefoods locations")
93
+
var summary StoreSummaryResponse
94
+
if err := json.NewDecoder(reader).Decode(&summary); err != nil {
95
+
return nil, fmt.Errorf("decode whole foods store summary: %w", err)
95
96
}
97
+
return &summary, nil
98
+
}
96
99
97
-
return summaries, nil
100
+
type loader struct {
101
+
cache cache.Cache
98
102
}
99
103
100
-
// StoreSummaryToLocation converts a whole food type intoa generic locaitn.
101
-
// Mostly vanilla except for prefixing name
102
-
func storeSummaryToLocation(summary StoreSummaryResponse) locationtypes.Location {
104
+
func (l *loader) Load(ctx context.Context, locationID string) (locationtypes.Location, error) {
105
+
summary, err := loadCachedStoreSummaryByID(ctx, l.cache, locationID)
106
+
if err != nil {
107
+
return locationtypes.Location{}, err
108
+
}
103
109
lat := summary.PrimaryLocation.Latitude
104
110
lon := summary.PrimaryLocation.Longitude
105
111
···
112
118
Lat: &lat,
113
119
Lon: &lon,
114
120
Chain: "wholefoods",
115
-
}
121
+
}, nil
116
122
}
+19
-19
internal/wholefoods/locations.go
+19
-19
internal/wholefoods/locations.go
···
7
7
8
8
"careme/internal/cache"
9
9
"careme/internal/config"
10
+
"careme/internal/locations/hydrator"
10
11
"careme/internal/locations/nearby"
12
+
"careme/internal/locations/storeindex"
13
+
11
14
locationtypes "careme/internal/locations/types"
12
15
)
13
16
···
17
20
18
21
type LocationBackend struct {
19
22
zipLookup centroidByZip
20
-
byID map[string]locationtypes.Location
23
+
spatial []locationtypes.Location
24
+
hydrator *hydrator.LazyHydrator
21
25
}
22
26
23
27
func NewLocationBackendFromConfig(ctx context.Context, cfg *config.Config, zipLookup centroidByZip) (*LocationBackend, error) {
···
37
41
return newLocationBackend(ctx, listCache, zipLookup)
38
42
}
39
43
40
-
func newLocationBackend(ctx context.Context, c cache.ListCache, zipLookup centroidByZip) (*LocationBackend, error) {
41
-
// Is this too much? should we just fetch a single blob that is all coordinates -> store ids and lazily fetch stores?
42
-
summaries, err := loadCachedStoreSummaries(ctx, c)
44
+
func newLocationBackend(ctx context.Context, c cache.Cache, zipLookup centroidByZip) (*LocationBackend, error) {
45
+
entries, err := storeindex.Load(ctx, c, LocationIndexCacheKey)
43
46
if err != nil {
44
-
return nil, err
47
+
return nil, fmt.Errorf("load wholefoods locations index: %w", err)
45
48
}
46
49
47
-
byID := make(map[string]locationtypes.Location, len(summaries))
48
-
for _, summary := range summaries {
49
-
loc := storeSummaryToLocation(*summary)
50
-
byID[loc.ID] = loc
50
+
spatial := make([]locationtypes.Location, 0, len(entries))
51
+
for _, entry := range entries {
52
+
spatial = append(spatial, entry.ToLocation())
51
53
}
52
54
53
55
return &LocationBackend{
54
56
zipLookup: zipLookup,
55
-
byID: byID,
57
+
spatial: spatial,
58
+
hydrator: hydrator.NewLazyHydrator(&loader{c}),
56
59
}, nil
57
60
}
58
61
···
65
68
return true
66
69
}
67
70
68
-
func (b *LocationBackend) GetLocationByID(_ context.Context, locationID string) (*locationtypes.Location, error) {
71
+
func (b *LocationBackend) GetLocationByID(ctx context.Context, locationID string) (*locationtypes.Location, error) {
69
72
normalized, ok := parseLocationID(locationID)
70
73
if !ok {
71
74
return nil, fmt.Errorf("whole foods location id %q is invalid", locationID)
72
75
}
73
76
74
-
loc, exists := b.byID[normalized]
75
-
if !exists {
76
-
return nil, fmt.Errorf("whole foods location %q not found", locationID)
77
+
loc, err := b.hydrator.Hydrate(ctx, normalized)
78
+
if err != nil {
79
+
return nil, err
77
80
}
78
81
79
82
copy := loc
···
81
84
}
82
85
83
86
func (b *LocationBackend) GetLocationsByZip(ctx context.Context, zipcode string) ([]locationtypes.Location, error) {
84
-
candidates := make([]locationtypes.Location, 0, len(b.byID))
85
-
for _, loc := range b.byID {
86
-
candidates = append(candidates, loc)
87
-
}
88
-
return nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, candidates, nearby.MaxLocationDistanceMiles), nil
87
+
candidates := nearby.FilterAndSortByZip(ctx, b.zipLookup, zipcode, b.spatial, nearby.MaxLocationDistanceMiles)
88
+
return storeindex.HydrateLocations(ctx, candidates, b.hydrator.Hydrate)
89
89
}
90
90
91
91
func parseLocationID(locationID string) (string, bool) {
+24
-8
internal/wholefoods/locations_test.go
+24
-8
internal/wholefoods/locations_test.go
···
16
16
if err := CacheStoreSummary(context.Background(), cacheStore, westlakeSummary()); err != nil {
17
17
t.Fatalf("CacheStoreSummary returned error: %v", err)
18
18
}
19
-
20
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
19
+
zipLookup := staticZIPLookup{
21
20
"98101": {Lat: 47.6101, Lon: -122.3344},
22
-
})
21
+
}
22
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
23
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
24
+
}
25
+
26
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
23
27
if err != nil {
24
28
t.Fatalf("newLocationBackend returned error: %v", err)
25
29
}
···
35
39
if loc.Name != "Whole Foods Westlake" || loc.ZipCode != "98121" || loc.Chain != "wholefoods" {
36
40
t.Fatalf("unexpected location: %+v", loc)
37
41
}
42
+
reader, err := cacheStore.Get(context.Background(), LocationIndexCacheKey)
43
+
if err != nil {
44
+
t.Fatalf("expected compact location index to be cached: %v", err)
45
+
}
46
+
_ = reader.Close()
38
47
}
39
48
40
49
func TestLocationBackendGetLocationsByZipUsesDistance(t *testing.T) {
···
47
56
if err := CacheStoreSummary(context.Background(), cacheStore, farStoreSummary()); err != nil {
48
57
t.Fatalf("cache far store summary: %v", err)
49
58
}
50
-
51
-
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{
59
+
zipLookup := staticZIPLookup{
52
60
"98101": {Lat: 47.6101, Lon: -122.3344},
53
-
})
61
+
}
62
+
if err := RebuildLocationIndex(context.Background(), cacheStore, zipLookup); err != nil {
63
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
64
+
}
65
+
66
+
backend, err := newLocationBackend(context.Background(), cacheStore, zipLookup)
54
67
if err != nil {
55
68
t.Fatalf("newLocationBackend returned error: %v", err)
56
69
}
···
79
92
}
80
93
if err := CacheStoreSummary(context.Background(), cacheStore, farStoreSummary()); err != nil {
81
94
t.Fatalf("cache far store summary: %v", err)
95
+
}
96
+
if err := RebuildLocationIndex(context.Background(), cacheStore, staticZIPLookup{}); err != nil {
97
+
t.Fatalf("RebuildLocationIndex returned error: %v", err)
82
98
}
83
99
84
100
backend, err := newLocationBackend(context.Background(), cacheStore, staticZIPLookup{})
···
104
120
if err == nil {
105
121
t.Fatal("expected newLocationBackend to return an error")
106
122
}
107
-
if !strings.Contains(err.Error(), "failed to load wholefoods locations") {
108
-
t.Fatalf("expected missing summaries error, got %v", err)
123
+
if !strings.Contains(err.Error(), "load wholefoods locations index") {
124
+
t.Fatalf("expected missing index error, got %v", err)
109
125
}
110
126
}
111
127