rss email digests over ssh because you're a cool kid
herald.dunkirk.sh
go
rss
rss-reader
ssh
charm
feat: prepared statements, cache control, and faster sorting
+169
-48
+2
-9
store/configs.go
+2
-9
store/configs.go
···
102
102
103
103
func (db *DB) GetConfig(ctx context.Context, userID int64, filename string) (*Config, error) {
104
104
var cfg Config
105
-
err := db.QueryRowContext(ctx,
106
-
`SELECT id, user_id, filename, email, cron_expr, digest, inline_content, raw_text, last_run, next_run, created_at
107
-
FROM configs WHERE user_id = ? AND filename = ?`,
108
-
userID, filename,
109
-
).Scan(&cfg.ID, &cfg.UserID, &cfg.Filename, &cfg.Email, &cfg.CronExpr, &cfg.Digest, &cfg.InlineContent, &cfg.RawText, &cfg.LastRun, &cfg.NextRun, &cfg.CreatedAt)
105
+
err := db.stmts.getConfig.QueryRowContext(ctx, userID, filename).Scan(&cfg.ID, &cfg.UserID, &cfg.Filename, &cfg.Email, &cfg.CronExpr, &cfg.Digest, &cfg.InlineContent, &cfg.RawText, &cfg.LastRun, &cfg.NextRun, &cfg.CreatedAt)
110
106
if err != nil {
111
107
return nil, err
112
108
}
···
168
164
}
169
165
170
166
func (db *DB) UpdateLastRun(ctx context.Context, configID int64, lastRun, nextRun time.Time) error {
171
-
_, err := db.ExecContext(ctx,
172
-
`UPDATE configs SET last_run = ?, next_run = ? WHERE id = ?`,
173
-
lastRun, nextRun, configID,
174
-
)
167
+
_, err := db.stmts.updateConfigRun.ExecContext(ctx, lastRun, nextRun, configID)
175
168
if err != nil {
176
169
return fmt.Errorf("update last run: %w", err)
177
170
}
+78
-1
store/db.go
+78
-1
store/db.go
···
10
10
11
11
type DB struct {
12
12
*sql.DB
13
+
stmts *preparedStmts
14
+
}
15
+
16
+
type preparedStmts struct {
17
+
markItemSeen *sql.Stmt
18
+
isItemSeen *sql.Stmt
19
+
getSeenItems *sql.Stmt
20
+
getConfig *sql.Stmt
21
+
updateConfigRun *sql.Stmt
22
+
updateFeedMeta *sql.Stmt
23
+
cleanupSeenItems *sql.Stmt
13
24
}
14
25
15
26
func Open(path string) (*DB, error) {
···
26
37
return nil, fmt.Errorf("ping database: %w", err)
27
38
}
28
39
29
-
store := &DB{db}
40
+
store := &DB{DB: db}
30
41
if err := store.migrate(); err != nil {
31
42
return nil, fmt.Errorf("migrate database: %w", err)
43
+
}
44
+
45
+
if err := store.prepareStatements(); err != nil {
46
+
return nil, fmt.Errorf("prepare statements: %w", err)
32
47
}
33
48
34
49
return store, nil
···
107
122
}
108
123
109
124
func (db *DB) Close() error {
125
+
if db.stmts != nil {
126
+
db.stmts.markItemSeen.Close()
127
+
db.stmts.isItemSeen.Close()
128
+
db.stmts.getSeenItems.Close()
129
+
db.stmts.getConfig.Close()
130
+
db.stmts.updateConfigRun.Close()
131
+
db.stmts.updateFeedMeta.Close()
132
+
db.stmts.cleanupSeenItems.Close()
133
+
}
110
134
return db.DB.Close()
135
+
}
136
+
137
+
func (db *DB) prepareStatements() error {
138
+
db.stmts = &preparedStmts{}
139
+
140
+
var err error
141
+
142
+
db.stmts.markItemSeen, err = db.Prepare(
143
+
`INSERT INTO seen_items (feed_id, guid, title, link) VALUES (?, ?, ?, ?)
144
+
ON CONFLICT(feed_id, guid) DO UPDATE SET title = excluded.title, link = excluded.link`)
145
+
if err != nil {
146
+
return fmt.Errorf("prepare markItemSeen: %w", err)
147
+
}
148
+
149
+
db.stmts.isItemSeen, err = db.Prepare(
150
+
`SELECT id FROM seen_items WHERE feed_id = ? AND guid = ?`)
151
+
if err != nil {
152
+
return fmt.Errorf("prepare isItemSeen: %w", err)
153
+
}
154
+
155
+
db.stmts.getSeenItems, err = db.Prepare(
156
+
`SELECT id, feed_id, guid, title, link, seen_at
157
+
FROM seen_items WHERE feed_id = ? ORDER BY seen_at DESC LIMIT ?`)
158
+
if err != nil {
159
+
return fmt.Errorf("prepare getSeenItems: %w", err)
160
+
}
161
+
162
+
db.stmts.getConfig, err = db.Prepare(
163
+
`SELECT id, user_id, filename, email, cron_expr, digest, inline_content, raw_text, last_run, next_run, created_at
164
+
FROM configs WHERE user_id = ? AND filename = ?`)
165
+
if err != nil {
166
+
return fmt.Errorf("prepare getConfig: %w", err)
167
+
}
168
+
169
+
db.stmts.updateConfigRun, err = db.Prepare(
170
+
`UPDATE configs SET last_run = ?, next_run = ? WHERE id = ?`)
171
+
if err != nil {
172
+
return fmt.Errorf("prepare updateConfigRun: %w", err)
173
+
}
174
+
175
+
db.stmts.updateFeedMeta, err = db.Prepare(
176
+
`UPDATE feeds SET last_fetched = ?, etag = ?, last_modified = ? WHERE id = ?`)
177
+
if err != nil {
178
+
return fmt.Errorf("prepare updateFeedMeta: %w", err)
179
+
}
180
+
181
+
db.stmts.cleanupSeenItems, err = db.Prepare(
182
+
`DELETE FROM seen_items WHERE seen_at < ?`)
183
+
if err != nil {
184
+
return fmt.Errorf("prepare cleanupSeenItems: %w", err)
185
+
}
186
+
187
+
return nil
111
188
}
112
189
113
190
func (db *DB) Migrate(ctx context.Context) error {
+1
-4
store/feeds.go
+1
-4
store/feeds.go
···
142
142
lmVal = sql.NullString{String: lastModified, Valid: true}
143
143
}
144
144
145
-
_, err := db.ExecContext(ctx,
146
-
`UPDATE feeds SET last_fetched = ?, etag = ?, last_modified = ? WHERE id = ?`,
147
-
time.Now(), etagVal, lmVal, feedID,
148
-
)
145
+
_, err := db.stmts.updateFeedMeta.ExecContext(ctx, time.Now(), etagVal, lmVal, feedID)
149
146
if err != nil {
150
147
return fmt.Errorf("update feed fetched: %w", err)
151
148
}
+4
-18
store/items.go
+4
-18
store/items.go
···
26
26
linkVal = sql.NullString{String: link, Valid: true}
27
27
}
28
28
29
-
_, err := db.ExecContext(ctx,
30
-
`INSERT INTO seen_items (feed_id, guid, title, link) VALUES (?, ?, ?, ?)
31
-
ON CONFLICT(feed_id, guid) DO UPDATE SET title = excluded.title, link = excluded.link`,
32
-
feedID, guid, titleVal, linkVal,
33
-
)
29
+
_, err := db.stmts.markItemSeen.ExecContext(ctx, feedID, guid, titleVal, linkVal)
34
30
if err != nil {
35
31
return fmt.Errorf("mark item seen: %w", err)
36
32
}
···
39
35
40
36
func (db *DB) IsItemSeen(ctx context.Context, feedID int64, guid string) (bool, error) {
41
37
var id int64
42
-
err := db.QueryRowContext(ctx,
43
-
`SELECT id FROM seen_items WHERE feed_id = ? AND guid = ?`,
44
-
feedID, guid,
45
-
).Scan(&id)
38
+
err := db.stmts.isItemSeen.QueryRowContext(ctx, feedID, guid).Scan(&id)
46
39
if err != nil {
47
40
if errors.Is(err, sql.ErrNoRows) {
48
41
return false, nil
···
73
66
}
74
67
75
68
func (db *DB) GetSeenItems(ctx context.Context, feedID int64, limit int) ([]*SeenItem, error) {
76
-
rows, err := db.QueryContext(ctx,
77
-
`SELECT id, feed_id, guid, title, link, seen_at
78
-
FROM seen_items WHERE feed_id = ? ORDER BY seen_at DESC LIMIT ?`,
79
-
feedID, limit,
80
-
)
69
+
rows, err := db.stmts.getSeenItems.QueryContext(ctx, feedID, limit)
81
70
if err != nil {
82
71
return nil, fmt.Errorf("query seen items: %w", err)
83
72
}
···
138
127
// CleanupOldSeenItems deletes seen items older than the specified duration
139
128
func (db *DB) CleanupOldSeenItems(ctx context.Context, olderThan time.Duration) (int64, error) {
140
129
cutoff := time.Now().Add(-olderThan)
141
-
result, err := db.ExecContext(ctx,
142
-
`DELETE FROM seen_items WHERE seen_at < ?`,
143
-
cutoff,
144
-
)
130
+
result, err := db.stmts.cleanupSeenItems.ExecContext(ctx, cutoff)
145
131
if err != nil {
146
132
return 0, fmt.Errorf("cleanup old seen items: %w", err)
147
133
}
+84
-16
web/handlers.go
+84
-16
web/handlers.go
···
170
170
PubDate string `xml:"pubDate"`
171
171
}
172
172
173
+
type rssItemWithTime struct {
174
+
rssItem
175
+
parsedTime time.Time
176
+
}
177
+
173
178
type rssChannel struct {
174
179
Title string `xml:"title"`
175
180
Link string `xml:"link"`
···
208
213
return
209
214
}
210
215
211
-
var items []rssItem
216
+
var items []rssItemWithTime
212
217
feeds, err := s.store.GetFeedsByConfig(ctx, cfg.ID)
213
218
if err != nil {
214
219
s.logger.Error("get feeds", "err", err)
···
222
227
continue
223
228
}
224
229
for _, item := range seenItems {
225
-
rItem := rssItem{
226
-
GUID: item.GUID,
227
-
PubDate: item.SeenAt.Format(time.RFC1123Z),
230
+
rItem := rssItemWithTime{
231
+
rssItem: rssItem{
232
+
GUID: item.GUID,
233
+
PubDate: item.SeenAt.Format(time.RFC1123Z),
234
+
},
235
+
parsedTime: item.SeenAt,
228
236
}
229
237
if item.Title.Valid {
230
238
rItem.Title = item.Title.String
···
237
245
}
238
246
239
247
sort.Slice(items, func(i, j int) bool {
240
-
ti, _ := time.Parse(time.RFC1123Z, items[i].PubDate)
241
-
tj, _ := time.Parse(time.RFC1123Z, items[j].PubDate)
242
-
return ti.After(tj)
248
+
return items[i].parsedTime.After(items[j].parsedTime)
243
249
})
244
250
245
251
if len(items) > 100 {
246
252
items = items[:100]
247
253
}
248
254
255
+
// Convert to rssItem for XML encoding
256
+
rssItems := make([]rssItem, len(items))
257
+
for i, item := range items {
258
+
rssItems[i] = item.rssItem
259
+
}
260
+
249
261
feed := rssFeed{
250
262
Version: "2.0",
251
263
Channel: rssChannel{
252
264
Title: "Herald - " + configFilename,
253
265
Link: s.origin + "/" + fingerprint + "/" + configFilename,
254
266
Description: "Feed for " + configFilename,
255
-
Items: items,
267
+
Items: rssItems,
256
268
},
257
269
}
258
270
271
+
// Add caching headers
259
272
w.Header().Set("Content-Type", "application/rss+xml; charset=utf-8")
273
+
w.Header().Set("Cache-Control", "public, max-age=300")
274
+
if cfg.LastRun.Valid {
275
+
etag := fmt.Sprintf(`"%s-%d"`, fingerprint[:8], cfg.LastRun.Time.Unix())
276
+
w.Header().Set("ETag", etag)
277
+
w.Header().Set("Last-Modified", cfg.LastRun.Time.UTC().Format(http.TimeFormat))
278
+
279
+
// Check If-None-Match
280
+
if match := r.Header.Get("If-None-Match"); match == etag {
281
+
w.WriteHeader(http.StatusNotModified)
282
+
return
283
+
}
284
+
285
+
// Check If-Modified-Since
286
+
if modSince := r.Header.Get("If-Modified-Since"); modSince != "" {
287
+
if t, err := http.ParseTime(modSince); err == nil && !cfg.LastRun.Time.After(t) {
288
+
w.WriteHeader(http.StatusNotModified)
289
+
return
290
+
}
291
+
}
292
+
}
293
+
260
294
w.Write([]byte(xml.Header))
261
295
enc := xml.NewEncoder(w)
262
296
enc.Indent("", " ")
···
278
312
DatePublished string `json:"date_published"`
279
313
}
280
314
315
+
type jsonFeedItemWithTime struct {
316
+
jsonFeedItem
317
+
parsedTime time.Time
318
+
}
319
+
281
320
func (s *Server) handleFeedJSON(w http.ResponseWriter, r *http.Request, fingerprint, configFilename string) {
282
321
ctx := r.Context()
283
322
···
303
342
return
304
343
}
305
344
306
-
var items []jsonFeedItem
345
+
var items []jsonFeedItemWithTime
307
346
feeds, err := s.store.GetFeedsByConfig(ctx, cfg.ID)
308
347
if err != nil {
309
348
s.logger.Error("get feeds", "err", err)
···
317
356
continue
318
357
}
319
358
for _, item := range seenItems {
320
-
jItem := jsonFeedItem{
321
-
ID: item.GUID,
322
-
DatePublished: item.SeenAt.Format(time.RFC3339),
359
+
jItem := jsonFeedItemWithTime{
360
+
jsonFeedItem: jsonFeedItem{
361
+
ID: item.GUID,
362
+
DatePublished: item.SeenAt.Format(time.RFC3339),
363
+
},
364
+
parsedTime: item.SeenAt,
323
365
}
324
366
if item.Title.Valid {
325
367
jItem.Title = item.Title.String
···
332
374
}
333
375
334
376
sort.Slice(items, func(i, j int) bool {
335
-
ti, _ := time.Parse(time.RFC3339, items[i].DatePublished)
336
-
tj, _ := time.Parse(time.RFC3339, items[j].DatePublished)
337
-
return ti.After(tj)
377
+
return items[i].parsedTime.After(items[j].parsedTime)
338
378
})
339
379
340
380
if len(items) > 100 {
341
381
items = items[:100]
342
382
}
343
383
384
+
// Convert to jsonFeedItem for JSON encoding
385
+
jsonItems := make([]jsonFeedItem, len(items))
386
+
for i, item := range items {
387
+
jsonItems[i] = item.jsonFeedItem
388
+
}
389
+
344
390
feed := jsonFeed{
345
391
Version: "https://jsonfeed.org/version/1.1",
346
392
Title: "Herald - " + configFilename,
347
393
HomePageURL: s.origin + "/" + fingerprint + "/" + configFilename,
348
394
FeedURL: s.origin + "/" + fingerprint + "/" + configFilename + ".json",
349
-
Items: items,
395
+
Items: jsonItems,
350
396
}
351
397
398
+
// Add caching headers
352
399
w.Header().Set("Content-Type", "application/feed+json; charset=utf-8")
400
+
w.Header().Set("Cache-Control", "public, max-age=300")
401
+
if cfg.LastRun.Valid {
402
+
etag := fmt.Sprintf(`"%s-%d"`, fingerprint[:8], cfg.LastRun.Time.Unix())
403
+
w.Header().Set("ETag", etag)
404
+
w.Header().Set("Last-Modified", cfg.LastRun.Time.UTC().Format(http.TimeFormat))
405
+
406
+
// Check If-None-Match
407
+
if match := r.Header.Get("If-None-Match"); match == etag {
408
+
w.WriteHeader(http.StatusNotModified)
409
+
return
410
+
}
411
+
412
+
// Check If-Modified-Since
413
+
if modSince := r.Header.Get("If-Modified-Since"); modSince != "" {
414
+
if t, err := http.ParseTime(modSince); err == nil && !cfg.LastRun.Time.After(t) {
415
+
w.WriteHeader(http.StatusNotModified)
416
+
return
417
+
}
418
+
}
419
+
}
420
+
353
421
enc := json.NewEncoder(w)
354
422
enc.SetIndent("", " ")
355
423
enc.Encode(feed)