+69 -9

.github/workflows/go.yml

reviewed

··· 1 1 # .github/workflows/ghcr-publish-pr-gate.yml 2 2 - name: Build and Publish 2 2 + name: Build and Publish 3 3 4 4 on: 5 5 push: 6 6 - branches: [master, main] # include both; trim as needed 6 6 + branches: [master, main] 7 7 pull_request: 8 8 branches: [master, main] 9 9 workflow_dispatch: 10 10 11 11 - # Cancel superseded runs (nice for PRs) 12 11 concurrency: 13 12 group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} 14 13 cancel-in-progress: true 15 14 16 15 jobs: 17 17 - gate: 18 18 - name: gate # this appears in Required checks UI as "<workflow> / gate" 16 16 + verify: 17 17 + name: verify 18 18 + runs-on: ubuntu-latest 19 19 + permissions: 20 20 + contents: read 21 21 + 22 22 + steps: 23 23 + - uses: actions/checkout@v4 24 24 + 25 25 + - uses: actions/setup-go@v5 26 26 + with: 27 27 + go-version-file: go.mod 28 28 + cache: true 29 29 + 30 30 + - name: Run Go tests 31 31 + run: go test ./... 32 32 + 33 33 + lint: 34 34 + name: lint 19 35 runs-on: ubuntu-latest 20 36 permissions: 21 37 contents: read 22 22 - packages: write # needed to push to GHCR on branch pushes 38 38 + 39 39 + steps: 40 40 + - uses: actions/checkout@v4 41 41 + 42 42 + - uses: actions/setup-go@v5 43 43 + with: 44 44 + go-version-file: go.mod 45 45 + cache: true 46 46 + 47 47 + - uses: golangci/golangci-lint-action@v9 48 48 + with: 49 49 + version: v2.11.3 50 50 + args: --timeout=5m 51 51 + 52 52 + publish: 53 53 + name: publish (${{ matrix.name }}) 54 54 + needs: [verify, lint] 55 55 + runs-on: ubuntu-latest 56 56 + permissions: 57 57 + contents: read 58 58 + packages: write 59 59 + strategy: 60 60 + fail-fast: false 61 61 + matrix: 62 62 + include: 63 63 + - name: careme 64 64 + cmd_path: ./cmd/careme 65 65 + image_suffix: "" 66 66 + - name: wholefoods 67 67 + cmd_path: ./cmd/wholefoods 68 68 + image_suffix: wholefoods 69 69 + - name: albertsons 70 70 + cmd_path: ./cmd/albertsons 71 71 + image_suffix: albertsons 72 72 + - name: publix 73 73 + cmd_path: ./cmd/publix 74 74 + image_suffix: publix 75 75 + - name: aldi 76 76 + cmd_path: ./cmd/aldi 77 77 + image_suffix: aldi 23 78 24 79 steps: 25 80 - uses: actions/checkout@v4 ··· 29 84 run: | 30 85 OWNER_LC=$(echo "${GITHUB_REPOSITORY_OWNER}" | tr '[:upper:]' '[:lower:]') 31 86 REPO_LC=$(echo "${GITHUB_REPOSITORY#*/}" | tr '[:upper:]' '[:lower:]') 32 32 - echo "IMAGE=ghcr.io/${OWNER_LC}/${REPO_LC}" >> "$GITHUB_OUTPUT" 87 87 + IMAGE="ghcr.io/${OWNER_LC}/${REPO_LC}" 88 88 + if [ -n "${{ matrix.image_suffix }}" ]; then 89 89 + IMAGE="${IMAGE}-${{ matrix.image_suffix }}" 90 90 + fi 91 91 + echo "IMAGE=${IMAGE}" >> "$GITHUB_OUTPUT" 33 92 echo "SHORT_SHA=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" 34 93 35 94 - name: Set up Buildx 36 95 uses: docker/setup-buildx-action@v3 37 96 38 38 - # Only needed when actually pushing 39 97 - name: Login to GHCR 40 98 if: github.event_name == 'push' 41 99 uses: docker/login-action@v3 ··· 48 106 uses: docker/build-push-action@v6 49 107 with: 50 108 context: . 51 51 - push: ${{ github.event_name == 'push' }} # false on PRs, true on branch pushes 109 109 + push: ${{ github.event_name == 'push' }} 110 110 + build-args: | 111 111 + CMD_PATH=${{ matrix.cmd_path }} 52 112 tags: | 53 113 ${{ steps.vars.outputs.IMAGE }}:latest 54 114 ${{ steps.vars.outputs.IMAGE }}:${{ steps.vars.outputs.SHORT_SHA }}

-22

.github/workflows/golangci-lint.yml

reviewed

··· 1 1 - name: golangci-lint 2 2 - 3 3 - on: 4 4 - push: 5 5 - branches: 6 6 - - main 7 7 - - master 8 8 - pull_request: 9 9 - 10 10 - jobs: 11 11 - golangci-lint: 12 12 - runs-on: ubuntu-latest 13 13 - steps: 14 14 - - uses: actions/checkout@v4 15 15 - - uses: actions/setup-go@v5 16 16 - with: 17 17 - go-version-file: go.mod 18 18 - cache: true 19 19 - - uses: golangci/golangci-lint-action@v9 20 20 - with: 21 21 - version: v2.8.0 22 22 - args: --timeout=5m

+4 -5

Dockerfile

reviewed

··· 2 2 # Stage 1: build 3 3 FROM golang:1.26-alpine AS builder 4 4 WORKDIR /src 5 5 + ARG CMD_PATH=./cmd/careme 5 6 # Enable module cache 6 7 COPY go.mod go.sum ./ 7 8 RUN go mod download 8 9 COPY . . 9 9 - RUN go test ./... -count=1 10 10 # Build static binary (no CGO) 11 11 - RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o careme ./cmd/careme 11 11 + RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o /out/app ${CMD_PATH} 12 12 13 13 # Stage 2: minimal runtime image 14 14 FROM gcr.io/distroless/static:nonroot 15 15 WORKDIR /workspace 16 16 - COPY --from=builder /src/careme /careme 16 16 + COPY --from=builder /out/app /app 17 17 # Copy CA certs (distroless already has them, included for clarity) 18 18 # COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 19 19 EXPOSE 8080 20 20 USER nonroot 21 21 - ENTRYPOINT ["/careme"] 22 22 - CMD ["-serve"] 21 21 + ENTRYPOINT ["/app"]

+8 -1

cmd/albertsons/main.go

reviewed

··· 3 3 import ( 4 4 "careme/internal/albertsons" 5 5 "careme/internal/cache" 6 6 + "careme/internal/logsetup" 6 7 "context" 7 8 "errors" 8 9 "flag" ··· 26 27 flag.IntVar(&delayMS, "delay-ms", 1000, "delay between store page requests in milliseconds") 27 28 flag.Parse() 28 29 30 30 + ctx := context.Background() 31 31 + closeLogger, err := logsetup.Configure(ctx) 32 32 + if err != nil { 33 33 + log.Fatalf("failed to configure logging: %v", err) 34 34 + } 35 35 + defer closeLogger() 36 36 + 29 37 chains, err := selectedChains(brands) 30 38 if err != nil { 31 39 log.Fatalf("failed to parse brands: %v", err) ··· 37 45 } 38 46 39 47 httpClient := &http.Client{Timeout: time.Duration(timeoutSec) * time.Second} 40 40 - ctx := context.Background() 41 48 delay := time.Duration(delayMS) * time.Millisecond 42 49 43 50 var synced int

+9 -1

cmd/aldi/main.go

reviewed

··· 3 3 import ( 4 4 "careme/internal/aldi" 5 5 "careme/internal/cache" 6 6 + "careme/internal/logsetup" 6 7 "context" 7 8 "flag" 8 9 "fmt" ··· 28 29 flag.IntVar(&timeoutSec, "timeout", 20, "HTTP timeout in seconds") 29 30 flag.Parse() 30 31 32 32 + ctx := context.Background() 33 33 + closeLogger, err := logsetup.Configure(ctx) 34 34 + if err != nil { 35 35 + log.Fatalf("failed to configure logging: %v", err) 36 36 + } 37 37 + defer closeLogger() 38 38 + 31 39 cacheStore, err := cache.EnsureCache(aldi.Container) 32 40 if err != nil { 33 41 log.Fatalf("failed to create cache: %v", err) ··· 36 44 httpClient := &http.Client{Timeout: time.Duration(timeoutSec) * time.Second} 37 45 client := aldi.NewClientWithBaseURL(baseURL, widgetKey, httpClient) 38 46 39 39 - synced, err := syncLocations(context.Background(), cacheStore, client) 47 47 + synced, err := syncLocations(ctx, cacheStore, client) 40 48 if err != nil { 41 49 log.Fatalf("failed to sync ALDI store summaries: %v", err) 42 50 }

+3 -45

cmd/careme/main.go

reviewed

··· 2 2 3 3 import ( 4 4 "careme/internal/config" 5 5 - "careme/internal/logsink" 5 5 + "careme/internal/logsetup" 6 6 "careme/internal/mail" 7 7 "careme/internal/static" 8 8 "careme/internal/templates" 9 9 "context" 10 10 _ "embed" 11 11 "flag" 12 12 - "fmt" 13 12 "log" 14 13 "log/slog" 15 14 "os" 16 16 - 17 17 - "github.com/openclosed-dev/slogan/appinsights" 18 18 - multi "github.com/samber/slog-multi" //this is getting a native version in newest golang 19 15 ) 20 16 21 21 - const appInsightsConnectionStringEnv = "APPLICATIONINSIGHTS_CONNECTION_STRING" 22 22 - 23 17 func main() { 24 18 var serve, mailer bool 25 19 var addr string ··· 42 36 log.Fatalf("failed to load configuration: %v", err) 43 37 } 44 38 45 45 - logcfg := logsink.ConfigFromEnv("logs") 46 46 - close, err := configureLogger(ctx, logcfg) 39 39 + close, err := logsetup.Configure(ctx) 47 40 if err != nil { 48 41 log.Fatalf("failed to configure logging: %v", err) 49 42 } ··· 64 57 return 65 58 } 66 59 67 67 - if err := runServer(cfg, logcfg, addr); err != nil { 60 60 + if err := runServer(cfg, addr); err != nil { 68 61 log.Fatalf("server error: %v", err) 69 62 } 70 63 } 71 71 - 72 72 - func configureLogger(ctx context.Context, logcfg logsink.Config) (func(), error) { 73 73 - handlers := make([]slog.Handler, 0, 3) 74 74 - var closers []func() //neat to be io.Closer 75 75 - if logcfg.Enabled() { 76 76 - handler, closer, err := logsink.NewJson(ctx, logcfg) 77 77 - if err != nil { 78 78 - return nil, fmt.Errorf("create logsink: %w", err) 79 79 - } 80 80 - handlers = append(handlers, handler) 81 81 - closers = append(closers, func() { 82 82 - if err := closer.Close(); err != nil { 83 83 - slog.Error("failed to close logsink", "error", err) 84 84 - } 85 85 - }) 86 86 - } 87 87 - if connectionString := os.Getenv(appInsightsConnectionStringEnv); connectionString != "" { 88 88 - handler, err := appinsights.NewHandler(connectionString, nil) 89 89 - if err != nil { 90 90 - return nil, fmt.Errorf("create app insights handler: %w", err) 91 91 - } 92 92 - handlers = append(handlers, handler) 93 93 - closers = append(closers, handler.Close) 94 94 - } 95 95 - 96 96 - close := func() { 97 97 - for _, closer := range closers { 98 98 - closer() 99 99 - } 100 100 - } 101 101 - 102 102 - handlers = append(handlers, slog.NewTextHandler(os.Stdout, nil)) 103 103 - slog.SetDefault(slog.New(multi.Fanout(handlers...))) 104 104 - return close, nil 105 105 - }

+2 -1

cmd/careme/middleware.go

reviewed

··· 1 1 package main 2 2 3 3 import ( 4 4 + "careme/internal/logsetup" 4 5 "errors" 5 6 "log/slog" 6 7 "net/http" ··· 81 82 } 82 83 83 84 func newAppInsightsTrackerFromEnv(next http.Handler) http.Handler { 84 84 - connectionString := os.Getenv(appInsightsConnectionStringEnv) 85 85 + connectionString := os.Getenv(logsetup.AppInsightsConnectionStringEnv) 85 86 if connectionString == "" { 86 87 return next 87 88 }

+1 -11

cmd/careme/web.go

reviewed

··· 8 8 "careme/internal/config" 9 9 "careme/internal/ingredients" 10 10 "careme/internal/locations" 11 11 - "careme/internal/logs" 12 12 - "careme/internal/logsink" 13 11 "careme/internal/recipes" 14 12 "careme/internal/seasons" 15 13 "careme/internal/sitemap" ··· 29 27 "time" 30 28 ) 31 29 32 32 - func runServer(cfg *config.Config, logsinkCfg logsink.Config, addr string) error { 30 30 + func runServer(cfg *config.Config, addr string) error { 33 31 cache, err := cache.MakeCache() 34 32 if err != nil { 35 33 return fmt.Errorf("failed to create cache: %w", err) ··· 77 75 ingredientsHandler := ingredients.NewHandler(cache) 78 76 ingredientsHandler.Register(adminMux) 79 77 mux.Handle("/admin/", admin.New(cfg, authClient).Enforce(http.StripPrefix("/admin", adminMux))) 80 80 - 81 81 - if logsinkCfg.Enabled() { 82 82 - logsHandler, err := logs.NewHandler(logsinkCfg) 83 83 - if err != nil { 84 84 - return fmt.Errorf("failed to create logs handler: %w", err) 85 85 - } 86 86 - logsHandler.Register(adminMux) 87 87 - } 88 78 89 79 mux.HandleFunc("/about", func(w http.ResponseWriter, r *http.Request) { 90 80 ctx := r.Context()

+9 -1

cmd/publix/main.go

reviewed

··· 2 2 3 3 import ( 4 4 "careme/internal/cache" 5 5 + "careme/internal/logsetup" 5 6 "careme/internal/publix" 6 7 "context" 7 8 "errors" ··· 46 47 flag.BoolVar(&resumeMissing, "resume-missing", true, "skip ids already recorded as missing") 47 48 flag.Parse() 48 49 50 50 + ctx := context.Background() 51 51 + closeLogger, err := logsetup.Configure(ctx) 52 52 + if err != nil { 53 53 + log.Fatalf("failed to configure logging: %v", err) 54 54 + } 55 55 + defer closeLogger() 56 56 + 49 57 if startID <= 0 { 50 58 log.Fatalf("start-id must be positive") 51 59 } ··· 61 69 httpClient := &http.Client{Timeout: time.Duration(timeoutSec) * time.Second} 62 70 client := publix.NewClientWithBaseURL(baseURL, httpClient) 63 71 64 64 - stats, err := syncStores(context.Background(), cacheStore, client, syncConfig{ 72 72 + stats, err := syncStores(ctx, cacheStore, client, syncConfig{ 65 73 startID: startID, 66 74 endID: endID, 67 75 delay: time.Duration(delayMS) * time.Millisecond,

+8 -1

cmd/wholefoods/main.go

reviewed

··· 2 2 3 3 import ( 4 4 "careme/internal/cache" 5 5 + "careme/internal/logsetup" 5 6 "careme/internal/wholefoods" 6 7 "context" 7 8 "errors" ··· 25 26 flag.IntVar(&timeoutSec, "timeout", 20, "HTTP timeout in seconds") 26 27 flag.Parse() 27 28 29 29 + ctx := context.Background() 30 30 + closeLogger, err := logsetup.Configure(ctx) 31 31 + if err != nil { 32 32 + log.Fatalf("failed to configure logging: %v", err) 33 33 + } 34 34 + defer closeLogger() 35 35 + 28 36 cacheStore, err := cache.EnsureCache(wholefoods.Container) 29 37 if err != nil { 30 38 log.Fatalf("failed to create cache: %v", err) ··· 33 41 httpClient := &http.Client{Timeout: time.Duration(timeoutSec) * time.Second} 34 42 client := wholefoods.NewClientWithBaseURL(baseURL, httpClient) 35 43 36 36 - ctx := context.Background() 37 44 refs, err := resolveStoreReferences(ctx, cacheStore, httpClient, sitemapURL) 38 45 if err != nil { 39 46 log.Fatalf("failed to resolve store references: %v", err)

+43

deploy/deploy.yaml

reviewed

··· 112 112 cpu: 500m 113 113 memory: 256Mi 114 114 --- 115 115 + apiVersion: batch/v1 116 116 + kind: CronJob 117 117 + metadata: 118 118 + name: wholefoods-scrape 119 119 + labels: 120 120 + app: careme 121 121 + spec: 122 122 + schedule: "0 6 * * 0" 123 123 + concurrencyPolicy: Forbid 124 124 + successfulJobsHistoryLimit: 2 125 125 + failedJobsHistoryLimit: 3 126 126 + jobTemplate: 127 127 + spec: 128 128 + backoffLimit: 1 129 129 + template: 130 130 + metadata: 131 131 + labels: 132 132 + app: careme 133 133 + job: wholefoods-scrape 134 134 + spec: 135 135 + restartPolicy: Never 136 136 + securityContext: 137 137 + runAsNonRoot: true 138 138 + runAsUser: 65532 139 139 + runAsGroup: 65532 140 140 + containers: 141 141 + - name: wholefoods 142 142 + image: ghcr.io/paulgmiller/careme-wholefoods:${IMAGE_TAG} 143 143 + imagePullPolicy: IfNotPresent 144 144 + envFrom: 145 145 + - secretRef: 146 146 + name: careme-secrets3 147 147 + env: 148 148 + - name: APPLICATIONINSIGHTS_CONNECTION_STRING 149 149 + value: "InstrumentationKey=a532fcc7-5098-4f44-8dde-ff2f32d6a59b;IngestionEndpoint=https://westus3-1.in.applicationinsights.azure.com/;LiveEndpoint=https://westus3.livediagnostics.monitor.azure.com/;ApplicationId=fdc94780-6135-4a29-980e-ab114a402e58" 150 150 + resources: 151 151 + requests: 152 152 + cpu: 50m 153 153 + memory: 64Mi 154 154 + limits: 155 155 + cpu: 500m 156 156 + memory: 256Mi 157 157 + --- 115 158 apiVersion: v1 116 159 kind: Service 117 160 metadata:

+1 -3

go.mod

reviewed

··· 1 1 module careme 2 2 3 3 - go 1.25 3 3 + go 1.26 4 4 5 5 tool github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen 6 6 ··· 14 14 github.com/microsoft/ApplicationInsights-Go v0.4.4 15 15 github.com/openai/openai-go/v3 v3.25.0 16 16 github.com/openclosed-dev/slogan v0.2.0 17 17 - github.com/samber/slog-multi v1.5.0 18 17 github.com/sendgrid/rest v2.6.9+incompatible 19 18 github.com/sendgrid/sendgrid-go v3.16.1+incompatible 20 19 golang.org/x/crypto v0.40.0 ··· 28 27 github.com/buger/jsonparser v1.1.1 // indirect 29 28 github.com/go-jose/go-jose/v3 v3.0.4 // indirect 30 29 github.com/gofrs/uuid v3.3.0+incompatible // indirect 31 31 - github.com/samber/slog-common v0.19.0 // indirect 32 30 github.com/stretchr/testify v1.11.1 // indirect 33 31 github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect 34 32 golang.org/x/sync v0.16.0 // indirect

-4

go.sum

reviewed

··· 129 129 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= 130 130 github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= 131 131 github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= 132 132 - github.com/samber/slog-common v0.19.0 h1:fNcZb8B2uOLooeYwFpAlKjkQTUafdjfqKcwcC89G9YI= 133 133 - github.com/samber/slog-common v0.19.0/go.mod h1:dTz+YOU76aH007YUU0DffsXNsGFQRQllPQh9XyNoA3M= 134 134 - github.com/samber/slog-multi v1.5.0 h1:UDRJdsdb0R5vFQFy3l26rpX3rL3FEPJTJ2yKVjoiT1I= 135 135 - github.com/samber/slog-multi v1.5.0/go.mod h1:im2Zi3mH/ivSY5XDj6LFcKToRIWPw1OcjSVSdXt+2d0= 136 132 github.com/sendgrid/rest v2.6.9+incompatible h1:1EyIcsNdn9KIisLW50MKwmSRSK+ekueiEMJ7NEoxJo0= 137 133 github.com/sendgrid/rest v2.6.9+incompatible/go.mod h1:kXX7q3jZtJXK5c5qK83bSGMdV6tsOE70KbHoqJls4lE= 138 134 github.com/sendgrid/sendgrid-go v3.16.1+incompatible h1:zWhTmB0Y8XCDzeWIm2/BIt1GjJohAA0p6hVEaDtHWWs=

-117

internal/logs/handler.go

reviewed

··· 1 1 - package logs 2 2 - 3 3 - import ( 4 4 - "careme/internal/logsink" 5 5 - "fmt" 6 6 - "log/slog" 7 7 - "net/http" 8 8 - "net/http/httputil" 9 9 - "net/url" 10 10 - "strconv" 11 11 - ) 12 12 - 13 13 - type handler struct { 14 14 - reader *Reader 15 15 - datasette http.Handler 16 16 - } 17 17 - 18 18 - func NewHandler(cfg logsink.Config) (*handler, error) { 19 19 - reader, err := NewReader(&cfg) 20 20 - if err != nil { 21 21 - return nil, fmt.Errorf("failed to create log reader: %w", err) 22 22 - } 23 23 - 24 24 - datasetteProxy, err := newDatasetteProxy() 25 25 - if err != nil { 26 26 - return nil, fmt.Errorf("failed to create datasette proxy: %w", err) 27 27 - } 28 28 - 29 29 - return &handler{ 30 30 - reader: reader, 31 31 - datasette: datasetteProxy, 32 32 - }, nil 33 33 - } 34 34 - 35 35 - // Register registers the log handler routes 36 36 - func (h *handler) Register(mux *http.ServeMux) { 37 37 - mux.HandleFunc("/logs", h.handleLogsPage) 38 38 - mux.HandleFunc("/api/logs", h.handleLogsAPI) 39 39 - mux.Handle("/datasette/", http.StripPrefix("/datasette", h.datasette)) 40 40 - } 41 41 - 42 42 - func (h *handler) handleLogsPage(w http.ResponseWriter, r *http.Request) { 43 43 - w.Header().Set("Content-Type", "text/html; charset=utf-8") 44 44 - w.Header().Set("X-Content-Type-Options", "nosniff") 45 45 - w.Header().Set("Cache-Control", "no-store") 46 46 - w.Header().Set("Content-Security-Policy", 47 47 - "default-src 'none'; "+ 48 48 - "script-src 'unsafe-inline'; "+ 49 49 - "base-uri 'none'; "+ 50 50 - "form-action 'none'; "+ 51 51 - "frame-ancestors 'none'; "+ 52 52 - "upgrade-insecure-requests;") 53 53 - 54 54 - page := `<!doctype html> 55 55 - <meta charset="utf-8" /> 56 56 - <title>Logs</title> 57 57 - <script> 58 58 - const api = new URL("/admin/api/logs", location.origin); 59 59 - const qs = new URLSearchParams(location.search); 60 60 - for (const k of ["hours"]) if (qs.has(k)) api.searchParams.set(k, qs.get(k)); 61 61 - 62 62 - const lite = new URL("/admin/datasette/", location.origin); 63 63 - lite.searchParams.set("json", api.toString()); 64 64 - 65 65 - location.replace(lite.toString()); 66 66 - </script>` 67 67 - 68 68 - _, err := w.Write([]byte(page)) 69 69 - if err != nil { 70 70 - slog.ErrorContext(r.Context(), "failed to write logs page", "error", err) 71 71 - } 72 72 - } 73 73 - 74 74 - func (h *handler) handleLogsAPI(w http.ResponseWriter, r *http.Request) { 75 75 - hoursStr := r.URL.Query().Get("hours") 76 76 - hours := 24 77 77 - if hoursStr != "" { 78 78 - if parsedHours, err := strconv.Atoi(hoursStr); err == nil && parsedHours > 0 { 79 79 - hours = parsedHours 80 80 - } 81 81 - } 82 82 - 83 83 - w.Header().Set("Access-Control-Allow-Origin", "*") 84 84 - w.Header().Set("Content-Type", "application/json") 85 85 - err := h.reader.GetLogs(r.Context(), hours, w) 86 86 - if err != nil { 87 87 - slog.ErrorContext(r.Context(), "failed to get logs", "error", err) 88 88 - http.Error(w, fmt.Sprintf("Failed to retrieve logs: %v", err), http.StatusInternalServerError) 89 89 - return 90 90 - } 91 91 - } 92 92 - 93 93 - func newDatasetteProxy() (http.Handler, error) { 94 94 - target, err := url.Parse("https://lite.datasette.io") 95 95 - if err != nil { 96 96 - return nil, err 97 97 - } 98 98 - 99 99 - proxy := newSingleHostProxy(target) 100 100 - proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, err error) { 101 101 - slog.ErrorContext(r.Context(), "datasette proxy request failed", "error", err) 102 102 - http.Error(w, "Datasette is unavailable", http.StatusBadGateway) 103 103 - } 104 104 - 105 105 - return proxy, nil 106 106 - } 107 107 - 108 108 - func newSingleHostProxy(target *url.URL) *httputil.ReverseProxy { 109 109 - proxy := httputil.NewSingleHostReverseProxy(target) 110 110 - originalDirector := proxy.Director 111 111 - proxy.Director = func(req *http.Request) { 112 112 - originalDirector(req) 113 113 - // Ensure upstream host routing matches the target domain. 114 114 - req.Host = target.Host 115 115 - } 116 116 - return proxy 117 117 - }

-68

internal/logs/handler_test.go

reviewed

··· 1 1 - package logs 2 2 - 3 3 - import ( 4 4 - "net/http" 5 5 - "net/http/httptest" 6 6 - "net/url" 7 7 - "strings" 8 8 - "testing" 9 9 - ) 10 10 - 11 11 - func TestHandleLogsPageRedirectsToLocalDatasette(t *testing.T) { 12 12 - t.Parallel() 13 13 - 14 14 - h := &handler{} 15 15 - req := httptest.NewRequest(http.MethodGet, "/admin/logs?hours=6", nil) 16 16 - rr := httptest.NewRecorder() 17 17 - h.handleLogsPage(rr, req) 18 18 - 19 19 - if rr.Code != http.StatusOK { 20 20 - t.Fatalf("expected status %d, got %d", http.StatusOK, rr.Code) 21 21 - } 22 22 - 23 23 - body := rr.Body.String() 24 24 - if !strings.Contains(body, `new URL("/admin/datasette/", location.origin)`) { 25 25 - t.Fatalf("expected local datasette URL in body, got: %s", body) 26 26 - } 27 27 - if strings.Contains(body, "https://lite.datasette.io/") { 28 28 - t.Fatalf("expected no direct lite.datasette.io redirect, got: %s", body) 29 29 - } 30 30 - } 31 31 - 32 32 - func TestNewDatasetteProxy(t *testing.T) { 33 33 - t.Parallel() 34 34 - 35 35 - proxy, err := newDatasetteProxy() 36 36 - if err != nil { 37 37 - t.Fatalf("expected proxy without error, got: %v", err) 38 38 - } 39 39 - if proxy == nil { 40 40 - t.Fatal("expected non-nil proxy") 41 41 - } 42 42 - } 43 43 - 44 44 - func TestNewSingleHostProxyRewritesHost(t *testing.T) { 45 45 - t.Parallel() 46 46 - 47 47 - target, err := url.Parse("https://example.test/sub") 48 48 - if err != nil { 49 49 - t.Fatalf("parse target URL: %v", err) 50 50 - } 51 51 - 52 52 - proxy := newSingleHostProxy(target) 53 53 - req := httptest.NewRequest(http.MethodGet, "http://localhost/admin/datasette/?json=http://localhost:8080/admin/api/logs", nil) 54 54 - proxy.Director(req) 55 55 - 56 56 - if req.Host != "example.test" { 57 57 - t.Fatalf("expected host example.test, got %s", req.Host) 58 58 - } 59 59 - if req.URL.Scheme != "https" { 60 60 - t.Fatalf("expected scheme https, got %s", req.URL.Scheme) 61 61 - } 62 62 - if req.URL.Host != "example.test" { 63 63 - t.Fatalf("expected url host example.test, got %s", req.URL.Host) 64 64 - } 65 65 - if req.URL.Path != "/sub/admin/datasette/" { 66 66 - t.Fatalf("expected path /sub/admin/datasette/, got %s", req.URL.Path) 67 67 - } 68 68 - }

-123

internal/logs/reader.go

reviewed

··· 1 1 - package logs 2 2 - 3 3 - import ( 4 4 - "careme/internal/logsink" 5 5 - "context" 6 6 - "errors" 7 7 - "fmt" 8 8 - "io" 9 9 - "log/slog" 10 10 - "time" 11 11 - 12 12 - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" 13 13 - ) 14 14 - 15 15 - // Reader reads logs from Azure Blob Storage 16 16 - type Reader struct { 17 17 - config *logsink.Config 18 18 - client *azblob.Client 19 19 - } 20 20 - 21 21 - // NewReader creates a new log reader 22 22 - func NewReader(cfg *logsink.Config) (*Reader, error) { 23 23 - if cfg.AccountName == "" || cfg.AccountKey == "" || cfg.Container == "" { 24 24 - return nil, errors.New("AccountName, AccountKey, and Container are required") 25 25 - } 26 26 - 27 27 - cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey) 28 28 - if err != nil { 29 29 - return nil, err 30 30 - } 31 31 - 32 32 - serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName) 33 33 - client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil) 34 34 - if err != nil { 35 35 - return nil, err 36 36 - } 37 37 - 38 38 - return &Reader{ 39 39 - config: cfg, 40 40 - client: client, 41 41 - }, nil 42 42 - } 43 43 - 44 44 - // GetLogs retrieves logs from the last N hours 45 45 - func (r *Reader) GetLogs(ctx context.Context, hours int, w io.Writer) error { 46 46 - if hours <= 0 { 47 47 - return errors.New("hours must be positive") 48 48 - } 49 49 - 50 50 - since := time.Now().Add(-time.Duration(hours) * time.Hour) 51 51 - 52 52 - // Generate date prefixes to query (covering the time range) 53 53 - datePrefixes := r.getDatePrefixes(since, time.Now()) 54 54 - 55 55 - // List blobs using date-based prefixes for efficiency 56 56 - for _, prefix := range datePrefixes { 57 57 - pager := r.client.NewListBlobsFlatPager(r.config.Container, &azblob.ListBlobsFlatOptions{ 58 58 - Prefix: &prefix, 59 59 - Include: azblob.ListBlobsInclude{Metadata: true}, 60 60 - }) 61 61 - 62 62 - for pager.More() { 63 63 - resp, err := pager.NextPage(ctx) 64 64 - if err != nil { 65 65 - return fmt.Errorf("failed to list blobs: %w", err) 66 66 - } 67 67 - 68 68 - for _, blobItem := range resp.Segment.BlobItems { 69 69 - // Skip blobs that haven't been modified in the time range (optimization) 70 70 - if blobItem.Properties.LastModified != nil && blobItem.Properties.LastModified.Before(since) { 71 71 - continue 72 72 - } 73 73 - 74 74 - // Read the blob content 75 75 - err := r.readBlobLogs(ctx, *blobItem.Name, w) 76 76 - if err != nil { 77 77 - // Log error but continue with other blobs 78 78 - slog.ErrorContext(ctx, "error reading blob", "blob", *blobItem.Name, "error", err) 79 79 - continue 80 80 - } 81 81 - 82 82 - } 83 83 - } 84 84 - } 85 85 - 86 86 - return nil 87 87 - } 88 88 - 89 89 - // getDatePrefixes generates date folder prefixes for the time range 90 90 - func (r *Reader) getDatePrefixes(since, until time.Time) []string { 91 91 - var prefixes []string 92 92 - current := since.UTC().Truncate(24 * time.Hour) 93 93 - end := until.UTC().Truncate(24 * time.Hour) 94 94 - 95 95 - for !current.After(end) { 96 96 - prefix := logsink.FormatDateFolder(current.Year(), int(current.Month()), current.Day()) + "/" 97 97 - prefixes = append(prefixes, prefix) 98 98 - current = current.Add(24 * time.Hour) 99 99 - } 100 100 - 101 101 - return prefixes 102 102 - } 103 103 - 104 104 - // readBlobLogs reads and parses log entries from a specific blob 105 105 - // can we parallelize this without busting the writer? 106 106 - func (r *Reader) readBlobLogs(ctx context.Context, blobName string, w io.Writer) error { 107 107 - 108 108 - blobClient := r.client.ServiceClient().NewContainerClient(r.config.Container).NewBlobClient(blobName) 109 109 - 110 110 - // Download the blob 111 111 - resp, err := blobClient.DownloadStream(ctx, nil) 112 112 - if err != nil { 113 113 - return fmt.Errorf("failed to download blob: %w", err) 114 114 - } 115 115 - defer func() { 116 116 - if err := resp.Body.Close(); err != nil { 117 117 - slog.ErrorContext(ctx, "failed to close log blob stream", "error", err, "blob", blobName) 118 118 - } 119 119 - }() 120 120 - 121 121 - _, err = io.Copy(w, resp.Body) 122 122 - return err 123 123 - }

-40

internal/logs/reader_test.go

reviewed

··· 1 1 - package logs 2 2 - 3 3 - import ( 4 4 - "testing" 5 5 - "time" 6 6 - ) 7 7 - 8 8 - func TestGetDatePrefixes(t *testing.T) { 9 9 - reader := &Reader{} 10 10 - 11 11 - // Test single day 12 12 - since := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC) 13 13 - until := time.Date(2024, 1, 15, 14, 0, 0, 0, time.UTC) 14 14 - prefixes := reader.getDatePrefixes(since, until) 15 15 - 16 16 - if len(prefixes) != 1 { 17 17 - t.Errorf("Expected 1 prefix for same day, got %d", len(prefixes)) 18 18 - } 19 19 - 20 20 - expected := "2024/01/15/" 21 21 - if prefixes[0] != expected { 22 22 - t.Errorf("Expected prefix %s, got %s", expected, prefixes[0]) 23 23 - } 24 24 - 25 25 - // Test multiple days 26 26 - since = time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC) 27 27 - until = time.Date(2024, 1, 17, 14, 0, 0, 0, time.UTC) 28 28 - prefixes = reader.getDatePrefixes(since, until) 29 29 - 30 30 - if len(prefixes) != 3 { 31 31 - t.Errorf("Expected 3 prefixes for 3 days, got %d", len(prefixes)) 32 32 - } 33 33 - 34 34 - expectedPrefixes := []string{"2024/01/15/", "2024/01/16/", "2024/01/17/"} 35 35 - for i, expected := range expectedPrefixes { 36 36 - if prefixes[i] != expected { 37 37 - t.Errorf("Expected prefix %s at index %d, got %s", expected, i, prefixes[i]) 38 38 - } 39 39 - } 40 40 - }

+32

internal/logsetup/logger.go

reviewed

··· 1 1 + package logsetup 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + "log/slog" 7 7 + "os" 8 8 + 9 9 + "github.com/openclosed-dev/slogan/appinsights" 10 10 + ) 11 11 + 12 12 + // just app insights for now. Giving up on logsink 13 13 + const AppInsightsConnectionStringEnv = "APPLICATIONINSIGHTS_CONNECTION_STRING" 14 14 + 15 15 + func Configure(ctx context.Context) (func(), error) { 16 16 + 17 17 + handlers := []slog.Handler{slog.NewTextHandler(os.Stdout, nil)} 18 18 + 19 19 + closeFn := func() {} //can be a list if we have multiple 20 20 + 21 21 + if connectionString := os.Getenv(AppInsightsConnectionStringEnv); connectionString != "" { 22 22 + handler, err := appinsights.NewHandler(connectionString, nil) 23 23 + if err != nil { 24 24 + return nil, fmt.Errorf("create app insights handler: %w", err) 25 25 + } 26 26 + handlers = append(handlers, handler) 27 27 + closeFn = handler.Close 28 28 + } 29 29 + 30 30 + slog.SetDefault(slog.New(slog.NewMultiHandler(handlers...))) 31 31 + return closeFn, nil 32 32 + }

-160

internal/logsink/appendblob.go

reviewed

··· 1 1 - package logsink 2 2 - 3 3 - import ( 4 4 - "bytes" 5 5 - "context" 6 6 - "errors" 7 7 - "fmt" 8 8 - "io" 9 9 - "log/slog" 10 10 - "net/url" 11 11 - "os" 12 12 - "sync" 13 13 - "time" 14 14 - 15 15 - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" 16 16 - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob" 17 17 - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" 18 18 - ) 19 19 - 20 20 - type Config struct { 21 21 - AccountName string 22 22 - AccountKey string 23 23 - Container string 24 24 - BlobName string // default hostname/podname 25 25 - FlushEvery time.Duration // default 2s 26 26 - } 27 27 - 28 28 - func ConfigFromEnv(container string) Config { 29 29 - return Config{ 30 30 - AccountName: os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"), 31 31 - AccountKey: os.Getenv("AZURE_STORAGE_PRIMARY_ACCOUNT_KEY"), 32 32 - Container: container, 33 33 - } 34 34 - } 35 35 - 36 36 - func (c Config) Enabled() bool { 37 37 - return c.AccountName != "" 38 38 - } 39 39 - 40 40 - type writer struct { 41 41 - ch chan []byte 42 42 - done chan bool 43 43 - wg sync.WaitGroup 44 44 - ticker *time.Ticker 45 45 - } 46 46 - 47 47 - var _ io.WriteCloser = &writer{} 48 48 - 49 49 - func New(ctx context.Context, cfg Config) (*writer, error) { 50 50 - if cfg.AccountName == "" || cfg.AccountKey == "" || cfg.Container == "" { 51 51 - return nil, errors.New("AccountName, AccountKey, and Container are required") 52 52 - } 53 53 - 54 54 - if cfg.BlobName == "" { 55 55 - cfg.BlobName, _ = os.Hostname() 56 56 - } 57 57 - 58 58 - // Add date-based folder structure: YYYY/MM/DD/hostname 59 59 - now := time.Now().UTC() 60 60 - dateFolder := FormatDateFolder(now.Year(), int(now.Month()), now.Day()) 61 61 - cfg.BlobName = dateFolder + "/" + cfg.BlobName 62 62 - 63 63 - if cfg.FlushEvery <= 0 { 64 64 - cfg.FlushEvery = 2 * time.Second 65 65 - } 66 66 - 67 67 - cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey) 68 68 - if err != nil { 69 69 - return nil, err 70 70 - } 71 71 - blobURL := "https://" + cfg.AccountName + ".blob.core.windows.net/" + 72 72 - url.PathEscape(cfg.Container) + "/" + cfg.BlobName // BlobName may include slashes; don’t path-escape it. 73 73 - 74 74 - ab, err := appendblob.NewClientWithSharedKeyCredential(blobURL, cred, nil) 75 75 - if err != nil { 76 76 - return nil, err 77 77 - } 78 78 - _, err = ab.Create(ctx, nil) // ignore error; maybe already exists 79 79 - if err != nil { 80 80 - if !bloberror.HasCode(err, bloberror.BlobAlreadyExists) { 81 81 - return nil, err 82 82 - } 83 83 - } 84 84 - 85 85 - h := &writer{ 86 86 - ch: make(chan []byte, 1024), // Buffered channel to hold log entries 87 87 - done: make(chan bool), //tie this in with context.Cancel? 88 88 - ticker: time.NewTicker(cfg.FlushEvery), 89 89 - } 90 90 - h.wg.Add(1) 91 91 - go h.loop(ctx, ab) 92 92 - return h, nil 93 93 - 94 94 - } 95 95 - 96 96 - func NewJson(ctx context.Context, cfg Config) (slog.Handler, io.Closer, error) { 97 97 - blobappender, err := New(ctx, cfg) 98 98 - if err != nil { 99 99 - return nil, nil, err 100 100 - } 101 101 - return slog.NewJSONHandler(blobappender, &slog.HandlerOptions{ 102 102 - AddSource: true, 103 103 - }), blobappender, nil 104 104 - } 105 105 - 106 106 - // Drain rest of logs. Will panic if called 107 107 - func (h *writer) Close() error { 108 108 - close(h.done) 109 109 - h.wg.Wait() 110 110 - h.ticker.Stop() 111 111 - return nil 112 112 - } 113 113 - 114 114 - func (h *writer) Write(p []byte) (n int, err error) { 115 115 - // Copy because slog reuses its buffers after Write returns. 116 116 - line := make([]byte, len(p)) 117 117 - copy(line, p) 118 118 - h.ch <- line 119 119 - //could err on closed but loggers just lose it anyways 120 120 - return len(p), nil 121 121 - } // internals 122 122 - 123 123 - func (h *writer) loop(ctx context.Context, ab *appendblob.Client) { 124 124 - defer h.wg.Done() 125 125 - var buf []byte 126 126 - flush := func() { 127 127 - if len(buf) == 0 { 128 128 - return 129 129 - } 130 130 - _, err := ab.AppendBlock(ctx, readSeekNopCloser{bytes.NewReader(buf)}, nil) 131 131 - if err != nil { 132 132 - fmt.Printf("error %s", err) 133 133 - } 134 134 - buf = buf[:0] //reset 135 135 - } 136 136 - 137 137 - for { 138 138 - select { 139 139 - case line := <-h.ch: 140 140 - buf = append(buf, line...) 141 141 - case <-h.ticker.C: 142 142 - flush() 143 143 - case <-h.done: 144 144 - //drain whats left 145 145 - for { 146 146 - select { 147 147 - case line := <-h.ch: 148 148 - buf = append(buf, line...) 149 149 - default: 150 150 - flush() 151 151 - return 152 152 - } 153 153 - } 154 154 - } 155 155 - } 156 156 - } 157 157 - 158 158 - type readSeekNopCloser struct{ io.ReadSeeker } 159 159 - 160 160 - func (r readSeekNopCloser) Close() error { return nil }

-12

internal/logsink/format.go

reviewed

··· 1 1 - package logsink 2 2 - 3 3 - import "fmt" 4 4 - 5 5 - // DateFolderFormat is the format string for organizing logs by date in blob storage 6 6 - // Format: YYYY/MM/DD/ 7 7 - const DateFolderFormat = "%d/%02d/%02d" 8 8 - 9 9 - // FormatDateFolder returns the date-based folder path for a given year, month, day 10 10 - func FormatDateFolder(year int, month int, day int) string { 11 11 - return fmt.Sprintf(DateFolderFormat, year, month, day) 12 12 - }