ai cooking
Merge pull request #76 from paulgmiller/logviewer
Logviewer
authored by
Paul Miller
and committed by
GitHub
f66a1537
cfa734bd
+301
-10
+3
-2
.gitignore
+3
-2
.gitignore
+4
-7
cmd/careme/main.go
+4
-7
cmd/careme/main.go
···
50
50
log.Fatalf("failed to load configuration: %v", err)
51
51
}
52
52
53
-
if _, ok := os.LookupEnv("AZURE_STORAGE_ACCOUNT_NAME"); ok {
54
-
handler, closer, err := logsink.NewJson(ctx, logsink.Config{
55
-
AccountName: os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"),
56
-
AccountKey: os.Getenv("AZURE_STORAGE_PRIMARY_ACCOUNT_KEY"),
57
-
Container: "logs",
58
-
})
53
+
logcfg := logsink.ConfigFromEnv("logs")
54
+
if logcfg.Enabled() {
55
+
handler, closer, err := logsink.NewJson(ctx, logcfg)
59
56
if err != nil {
60
57
log.Fatalf("failed to create logsink: %v", err)
61
58
}
···
75
72
}
76
73
77
74
if serve {
78
-
if err := runServer(cfg, addr); err != nil {
75
+
if err := runServer(cfg, logcfg, addr); err != nil {
79
76
log.Fatalf("server error: %v", err)
80
77
}
81
78
return
+11
-1
cmd/careme/web.go
+11
-1
cmd/careme/web.go
···
5
5
"careme/internal/config"
6
6
"careme/internal/html"
7
7
"careme/internal/locations"
8
+
"careme/internal/logs"
9
+
"careme/internal/logsink"
8
10
"careme/internal/recipes"
9
11
"careme/internal/templates"
10
12
"careme/internal/users"
···
24
26
25
27
const sessionDuration = 365 * 24 * time.Hour
26
28
27
-
func runServer(cfg *config.Config, addr string) error {
29
+
func runServer(cfg *config.Config, logsinkCfg logsink.Config, addr string) error {
28
30
29
31
cache, err := cache.MakeCache()
30
32
if err != nil {
···
51
53
52
54
recipeHandler := recipes.NewHandler(cfg, userStorage, generator, clarityScript, locationserver)
53
55
recipeHandler.Register(mux)
56
+
57
+
if logsinkCfg.Enabled() {
58
+
logsHandler, err := logs.NewHandler(logsinkCfg)
59
+
if err != nil {
60
+
return fmt.Errorf("failed to create logs handler: %w", err)
61
+
}
62
+
logsHandler.Register(mux)
63
+
}
54
64
55
65
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
56
66
ctx := r.Context()
+91
internal/logs/handler.go
+91
internal/logs/handler.go
···
1
+
// TODO merge with log sink
2
+
package logs
3
+
4
+
import (
5
+
"careme/internal/logsink"
6
+
"context"
7
+
"fmt"
8
+
"log/slog"
9
+
"net/http"
10
+
"strconv"
11
+
)
12
+
13
+
// Handler handles HTTP requests for log viewing
14
+
type handler struct {
15
+
reader *Reader
16
+
}
17
+
18
+
// NewHandler creates a new logs HTTP handler
19
+
func NewHandler(cfg logsink.Config) (*handler, error) {
20
+
// Only create reader if Azure credentials are available
21
+
reader, err := NewReader(context.Background(), &cfg)
22
+
if err != nil {
23
+
return nil, fmt.Errorf("failed to create log reader: %w", err)
24
+
}
25
+
26
+
return &handler{
27
+
reader: reader,
28
+
}, nil
29
+
}
30
+
31
+
// Register registers the log handler routes
32
+
func (h *handler) Register(mux *http.ServeMux) {
33
+
mux.HandleFunc("/logs", h.handleLogsPage)
34
+
mux.HandleFunc("/api/logs", h.handleLogsAPI)
35
+
}
36
+
37
+
func (h *handler) handleLogsPage(w http.ResponseWriter, r *http.Request) {
38
+
w.Header().Set("Content-Type", "text/html; charset=utf-8")
39
+
w.Header().Set("X-Content-Type-Options", "nosniff")
40
+
w.Header().Set("Cache-Control", "no-store")
41
+
w.Header().Set("Content-Security-Policy",
42
+
"default-src 'none'; "+
43
+
"script-src 'unsafe-inline'; "+
44
+
"base-uri 'none'; "+
45
+
"form-action 'none'; "+
46
+
"frame-ancestors 'none'; "+
47
+
"upgrade-insecure-requests;")
48
+
49
+
_, err := w.Write([]byte(`<!doctype html>
50
+
<meta charset="utf-8" />
51
+
<title>Logs</title>
52
+
<script>
53
+
const api = new URL("/api/logs", location.origin);
54
+
const qs = new URLSearchParams(location.search);
55
+
for (const k of ["hours"]) if (qs.has(k)) api.searchParams.set(k, qs.get(k));
56
+
57
+
const lite = new URL("https://lite.datasette.io/");
58
+
lite.searchParams.set("json", api.toString());
59
+
// Optional: turn off analytics
60
+
// lite.searchParams.set("analytics", "off");
61
+
62
+
location.replace(lite.toString());
63
+
</script>`))
64
+
if err != nil {
65
+
slog.ErrorContext(r.Context(), "failed to write logs page", "error", err)
66
+
}
67
+
}
68
+
69
+
// handleLogsAPI serves the logs as JSON
70
+
func (h *handler) handleLogsAPI(w http.ResponseWriter, r *http.Request) {
71
+
72
+
// Parse hours parameter
73
+
hoursStr := r.URL.Query().Get("hours")
74
+
hours := 24 // default
75
+
if hoursStr != "" {
76
+
if h, err := strconv.Atoi(hoursStr); err == nil && h > 0 {
77
+
hours = h
78
+
}
79
+
}
80
+
81
+
// Get logs
82
+
// Return as JSON // kinda?
83
+
w.Header().Set("Content-Type", "application/json")
84
+
err := h.reader.GetLogs(r.Context(), hours, w)
85
+
if err != nil {
86
+
slog.ErrorContext(r.Context(), "failed to get logs", "error", err)
87
+
http.Error(w, fmt.Sprintf("Failed to retrieve logs: %v", err), http.StatusInternalServerError)
88
+
return
89
+
}
90
+
91
+
}
+119
internal/logs/reader.go
+119
internal/logs/reader.go
···
1
+
package logs
2
+
3
+
import (
4
+
"careme/internal/logsink"
5
+
"context"
6
+
"errors"
7
+
"fmt"
8
+
"io"
9
+
"log/slog"
10
+
"time"
11
+
12
+
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
13
+
)
14
+
15
+
// Reader reads logs from Azure Blob Storage
16
+
type Reader struct {
17
+
config *logsink.Config
18
+
client *azblob.Client
19
+
}
20
+
21
+
// NewReader creates a new log reader
22
+
func NewReader(ctx context.Context, cfg *logsink.Config) (*Reader, error) {
23
+
if cfg.AccountName == "" || cfg.AccountKey == "" || cfg.Container == "" {
24
+
return nil, errors.New("AccountName, AccountKey, and Container are required")
25
+
}
26
+
27
+
cred, err := azblob.NewSharedKeyCredential(cfg.AccountName, cfg.AccountKey)
28
+
if err != nil {
29
+
return nil, err
30
+
}
31
+
32
+
serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", cfg.AccountName)
33
+
client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, cred, nil)
34
+
if err != nil {
35
+
return nil, err
36
+
}
37
+
38
+
return &Reader{
39
+
config: cfg,
40
+
client: client,
41
+
}, nil
42
+
}
43
+
44
+
// GetLogs retrieves logs from the last N hours
45
+
func (r *Reader) GetLogs(ctx context.Context, hours int, w io.Writer) error {
46
+
if hours <= 0 {
47
+
return errors.New("hours must be positive")
48
+
}
49
+
50
+
since := time.Now().Add(-time.Duration(hours) * time.Hour)
51
+
52
+
// Generate date prefixes to query (covering the time range)
53
+
datePrefixes := r.getDatePrefixes(since, time.Now())
54
+
55
+
// List blobs using date-based prefixes for efficiency
56
+
for _, prefix := range datePrefixes {
57
+
pager := r.client.NewListBlobsFlatPager(r.config.Container, &azblob.ListBlobsFlatOptions{
58
+
Prefix: &prefix,
59
+
Include: azblob.ListBlobsInclude{Metadata: true},
60
+
})
61
+
62
+
for pager.More() {
63
+
resp, err := pager.NextPage(ctx)
64
+
if err != nil {
65
+
return fmt.Errorf("failed to list blobs: %w", err)
66
+
}
67
+
68
+
for _, blobItem := range resp.Segment.BlobItems {
69
+
// Skip blobs that haven't been modified in the time range (optimization)
70
+
if blobItem.Properties.LastModified != nil && blobItem.Properties.LastModified.Before(since) {
71
+
continue
72
+
}
73
+
74
+
// Read the blob content
75
+
err := r.readBlobLogs(ctx, *blobItem.Name, w)
76
+
if err != nil {
77
+
// Log error but continue with other blobs
78
+
slog.ErrorContext(ctx, "error reading blob", "blob", *blobItem.Name, "error", err)
79
+
continue
80
+
}
81
+
82
+
}
83
+
}
84
+
}
85
+
86
+
return nil
87
+
}
88
+
89
+
// getDatePrefixes generates date folder prefixes for the time range
90
+
func (r *Reader) getDatePrefixes(since, until time.Time) []string {
91
+
var prefixes []string
92
+
current := since.UTC().Truncate(24 * time.Hour)
93
+
end := until.UTC().Truncate(24 * time.Hour)
94
+
95
+
for !current.After(end) {
96
+
prefix := logsink.FormatDateFolder(current.Year(), int(current.Month()), current.Day()) + "/"
97
+
prefixes = append(prefixes, prefix)
98
+
current = current.Add(24 * time.Hour)
99
+
}
100
+
101
+
return prefixes
102
+
}
103
+
104
+
// readBlobLogs reads and parses log entries from a specific blob
105
+
// can we parallelize this without busting the writer?
106
+
func (r *Reader) readBlobLogs(ctx context.Context, blobName string, w io.Writer) error {
107
+
108
+
blobClient := r.client.ServiceClient().NewContainerClient(r.config.Container).NewBlobClient(blobName)
109
+
110
+
// Download the blob
111
+
resp, err := blobClient.DownloadStream(ctx, nil)
112
+
if err != nil {
113
+
return fmt.Errorf("failed to download blob: %w", err)
114
+
}
115
+
defer resp.Body.Close()
116
+
117
+
_, err = io.Copy(w, resp.Body)
118
+
return err
119
+
}
+40
internal/logs/reader_test.go
+40
internal/logs/reader_test.go
···
1
+
package logs
2
+
3
+
import (
4
+
"testing"
5
+
"time"
6
+
)
7
+
8
+
func TestGetDatePrefixes(t *testing.T) {
9
+
reader := &Reader{}
10
+
11
+
// Test single day
12
+
since := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
13
+
until := time.Date(2024, 1, 15, 14, 0, 0, 0, time.UTC)
14
+
prefixes := reader.getDatePrefixes(since, until)
15
+
16
+
if len(prefixes) != 1 {
17
+
t.Errorf("Expected 1 prefix for same day, got %d", len(prefixes))
18
+
}
19
+
20
+
expected := "2024/01/15/"
21
+
if prefixes[0] != expected {
22
+
t.Errorf("Expected prefix %s, got %s", expected, prefixes[0])
23
+
}
24
+
25
+
// Test multiple days
26
+
since = time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC)
27
+
until = time.Date(2024, 1, 17, 14, 0, 0, 0, time.UTC)
28
+
prefixes = reader.getDatePrefixes(since, until)
29
+
30
+
if len(prefixes) != 3 {
31
+
t.Errorf("Expected 3 prefixes for 3 days, got %d", len(prefixes))
32
+
}
33
+
34
+
expectedPrefixes := []string{"2024/01/15/", "2024/01/16/", "2024/01/17/"}
35
+
for i, expected := range expectedPrefixes {
36
+
if prefixes[i] != expected {
37
+
t.Errorf("Expected prefix %s at index %d, got %s", expected, i, prefixes[i])
38
+
}
39
+
}
40
+
}
+17
internal/logsink/appendblob.go
+17
internal/logsink/appendblob.go
···
25
25
FlushEvery time.Duration // default 2s
26
26
}
27
27
28
+
func ConfigFromEnv(container string) Config {
29
+
return Config{
30
+
AccountName: os.Getenv("AZURE_STORAGE_ACCOUNT_NAME"),
31
+
AccountKey: os.Getenv("AZURE_STORAGE_PRIMARY_ACCOUNT_KEY"),
32
+
Container: container,
33
+
}
34
+
}
35
+
36
+
func (c Config) Enabled() bool {
37
+
return c.AccountName != ""
38
+
}
39
+
28
40
type writer struct {
29
41
ch chan []byte
30
42
done chan bool
···
42
54
if cfg.BlobName == "" {
43
55
cfg.BlobName, _ = os.Hostname()
44
56
}
57
+
58
+
// Add date-based folder structure: YYYY/MM/DD/hostname
59
+
now := time.Now().UTC()
60
+
dateFolder := FormatDateFolder(now.Year(), int(now.Month()), now.Day())
61
+
cfg.BlobName = dateFolder + "/" + cfg.BlobName
45
62
46
63
if cfg.FlushEvery <= 0 {
47
64
cfg.FlushEvery = 2 * time.Second
+12
internal/logsink/format.go
+12
internal/logsink/format.go
···
1
+
package logsink
2
+
3
+
import "fmt"
4
+
5
+
// DateFolderFormat is the format string for organizing logs by date in blob storage
6
+
// Format: YYYY/MM/DD/
7
+
const DateFolderFormat = "%d/%02d/%02d"
8
+
9
+
// FormatDateFolder returns the date-based folder path for a given year, month, day
10
+
func FormatDateFolder(year int, month int, day int) string {
11
+
return fmt.Sprintf(DateFolderFormat, year, month, day)
12
+
}
+4
internal/templates/home.html
+4
internal/templates/home.html
···
65
65
class="inline-flex items-center justify-center rounded-lg bg-brand-600 px-4 py-2.5 text-sm font-semibold text-white shadow-md transition hover:bg-brand-700 focus:outline-none focus:ring-2 focus:ring-brand-400 focus:ring-offset-2">
66
66
Past Recipes
67
67
</a>
68
+
<a href="/logs"
69
+
class="inline-flex items-center justify-center rounded-lg bg-gray-600 px-4 py-2.5 text-sm font-semibold text-white shadow-md transition hover:bg-gray-700 focus:outline-none focus:ring-2 focus:ring-gray-400 focus:ring-offset-2">
70
+
View Logs
71
+
</a>
68
72
</div>
69
73
<form method="GET" action="/locations" class="mt-6 space-y-2">
70
74
<label for="zip" class="block text-sm font-medium text-gray-700">