ai cooking
invert caching and multigrader (#533)
* invert caching and multigrader
* save partial grades
* save partials
---------
Co-authored-by: paul miller <paul.miller>
authored by
Paul Miller
paul miller
and committed by
GitHub
5e5811e0
c950dc68
+27
-24
+18
-10
internal/ingredients/grading/cache.go
+18
-10
internal/ingredients/grading/cache.go
···
5
5
"errors"
6
6
"fmt"
7
7
"log/slog"
8
+
"sync"
8
9
9
10
"careme/internal/ai"
10
11
"careme/internal/cache"
···
82
83
}
83
84
84
85
gradedIngredients, err := c.grader.GradeIngredients(ctx, missingIngredients)
85
-
if err != nil {
86
-
return nil, err
87
-
}
88
-
if len(gradedIngredients) != len(missingIngredients) {
89
-
return nil, fmt.Errorf("ingredient grader returned %d ingredients for %d inputs", len(gradedIngredients), len(missingIngredients))
90
-
}
91
86
87
+
// might get partial results back save those.
88
+
var wg sync.WaitGroup
92
89
for _, gradedIngredient := range gradedIngredients {
93
90
if gradedIngredient.Grade == nil {
94
91
return nil, fmt.Errorf("ingredient grader returned nil grade for %q", ingredientLabel(gradedIngredient))
95
92
}
96
-
key := cacheKey(c.cacheVersion + "/" + ingredientHash(gradedIngredient))
97
-
if err := c.store.Save(ctx, key, &gradedIngredient); err != nil {
98
-
slog.ErrorContext(ctx, "failed to cache ingredient grade", "key", key, "ingredient", ingredientLabel(gradedIngredient), "error", err)
99
-
}
100
93
results = append(results, gradedIngredient)
94
+
wg.Go(func() {
95
+
ctx := context.WithoutCancel(ctx)
96
+
key := cacheKey(c.cacheVersion + "/" + ingredientHash(gradedIngredient))
97
+
if err := c.store.Save(ctx, key, &gradedIngredient); err != nil {
98
+
slog.ErrorContext(ctx, "failed to cache ingredient grade", "key", key, "ingredient", ingredientLabel(gradedIngredient), "error", err)
99
+
}
100
+
})
101
+
}
102
+
wg.Wait()
103
+
if err != nil {
104
+
return nil, err
105
+
}
106
+
107
+
if len(gradedIngredients) != len(missingIngredients) {
108
+
return nil, fmt.Errorf("ingredient grader returned %d ingredients for %d inputs", len(gradedIngredients), len(missingIngredients))
101
109
}
102
110
103
111
return results, nil
+8
-11
internal/ingredients/grading/manager.go
+8
-11
internal/ingredients/grading/manager.go
···
35
35
}
36
36
37
37
type multiGrader struct {
38
-
grader grader
38
+
grader baseGrader
39
+
}
40
+
41
+
func (m *multiGrader) CacheVersion() string {
42
+
return m.grader.CacheVersion()
39
43
}
40
44
41
45
func NewManager(cfg *config.Config, c cache.ListCache) grader {
···
43
47
return rubberstamp{}
44
48
}
45
49
base := ai.NewIngredientGrader(cfg.AI.APIKey, cfg.IngredientGrading.Model)
46
-
return &multiGrader{
47
-
grader: newCachingGrader(base, NewStore(c)),
48
-
}
50
+
return newCachingGrader(&multiGrader{grader: base}, NewStore(c))
49
51
}
50
52
51
53
func (m *multiGrader) GradeIngredients(ctx context.Context, ingredients []ai.InputIngredient) ([]ai.InputIngredient, error) {
···
54
56
}
55
57
56
58
// we assume dedupe before thing come in here
57
-
58
59
batches := lo.Chunk(ingredients, ingredientGradeBatchSize)
59
-
graded, err := parallelism.Flatten(batches, func(batch []ai.InputIngredient) ([]ai.InputIngredient, error) {
60
+
// return partial results so we can cache them
61
+
return parallelism.Flatten(batches, func(batch []ai.InputIngredient) ([]ai.InputIngredient, error) {
60
62
return m.grader.GradeIngredients(ctx, batch)
61
63
})
62
-
if err != nil {
63
-
// will have cached these
64
-
return nil, err
65
-
}
66
-
return graded, nil
67
64
}
68
65
69
66
func ingredientHash(ingredient ai.InputIngredient) string {
+1
-3
internal/ingredients/grading/manager_test.go
+1
-3
internal/ingredients/grading/manager_test.go
···
103
103
func TestMultiGraderBatchesUniqueIngredientsInChunksOf30(t *testing.T) {
104
104
cacheStore := NewStore(cache.NewInMemoryCache())
105
105
backend := &stubGradeBackend{}
106
-
manager := &multiGrader{
107
-
grader: newCachingGrader(backend, cacheStore),
108
-
}
106
+
manager := newCachingGrader(&multiGrader{backend}, cacheStore)
109
107
110
108
ingredients := make([]ai.InputIngredient, 65)
111
109
for i := range ingredients {