this repo has no description
feat: server side caching
This commit implements server side caching for the preview cards. This should
reduce load time quite a bit, once at least one person has loaded it. Expiry is
24 hours.
There is also an endpoint at /api/caching/invalidate to clear the cache.
+259
-5
+8
README.md
+8
README.md
···
252
252
[MySQL] 3. Verify user has permissions: GRANT ALL ON tumble.* TO 'tumble'@'localhost'
253
253
```
254
254
255
+
### Caching
256
+
257
+
Link previews are cached in the database to reduce external requests.
258
+
259
+
- `caching.enabled`: Set to `false` to disable server-side caching.
260
+
- To invalidate a cache entry manually:
261
+
`GET /api/caching/invalidate?url=<encoded_url>`
262
+
255
263
## Bugs
256
264
257
265
* fix user-agent being hardy for link verification
+1
cmd/tumble/main.go
+1
cmd/tumble/main.go
···
167
167
mux.HandleFunc("/irclink/", h.IRCLinkHandler) // Handles /irclink/?id and posts
168
168
169
169
mux.HandleFunc("/ogpreview.cgi", h.OGPreviewHandler)
170
+
mux.HandleFunc("/api/caching/invalidate", h.InvalidateCacheHandler)
170
171
mux.HandleFunc("/buttons/", h.ButtonHandler) // Handle /buttons/ with ButtonHandler (landing + result)
171
172
mux.HandleFunc("/buttons/button.cgi", h.ButtonHandler) // Legacy explicit path
172
173
+3
conf/config.yaml
+3
conf/config.yaml
+36
internal/assets/openapi.json
+36
internal/assets/openapi.json
···
117
117
}
118
118
}
119
119
},
120
+
"/api/caching/invalidate": {
121
+
"get": {
122
+
"summary": "Invalidate Link Preview Cache",
123
+
"description": "Removes a URL from the server-side link preview cache.",
124
+
"parameters": [
125
+
{
126
+
"name": "url",
127
+
"in": "query",
128
+
"description": "The URL to invalidate",
129
+
"required": true,
130
+
"schema": {
131
+
"type": "string"
132
+
}
133
+
}
134
+
],
135
+
"responses": {
136
+
"200": {
137
+
"description": "Cache Invalidated",
138
+
"content": {
139
+
"application/json": {
140
+
"schema": {
141
+
"type": "object",
142
+
"properties": {
143
+
"status": { "type": "string" },
144
+
"message": { "type": "string" }
145
+
}
146
+
}
147
+
}
148
+
}
149
+
},
150
+
"400": {
151
+
"description": "Missing URL"
152
+
}
153
+
}
154
+
}
155
+
},
120
156
"/ogpreview.cgi": {
121
157
"get": {
122
158
"summary": "Get OpenGraph Preview",
+6
internal/config/config.go
+6
internal/config/config.go
···
17
17
Port string `yaml:"port" mapstructure:"port"`
18
18
Mode string `yaml:"mode" mapstructure:"mode"`
19
19
Logging Logging `yaml:"logging" mapstructure:"logging"`
20
+
Caching Caching `yaml:"caching" mapstructure:"caching"`
21
+
}
22
+
23
+
type Caching struct {
24
+
Enabled bool `yaml:"enabled" mapstructure:"enabled"`
20
25
}
21
26
22
27
type Logging struct {
···
33
38
v.SetDefault("mode", "production")
34
39
v.SetDefault("logging.level", "info")
35
40
v.SetDefault("logging.output", "stdout")
41
+
v.SetDefault("caching.enabled", true)
36
42
37
43
// Environment Variables
38
44
v.SetEnvPrefix("TUMBLE")
+30
-1
internal/data/gorm_store.go
+30
-1
internal/data/gorm_store.go
···
26
26
}
27
27
28
28
func (s *GormStore) Bootstrap(ctx context.Context) error {
29
-
return s.db.AutoMigrate(&IRCLink{}, &Image{}, &Quote{})
29
+
return s.db.AutoMigrate(&IRCLink{}, &Image{}, &Quote{}, &LinkPreview{})
30
30
}
31
31
32
32
func (s *GormStore) GetRecentIRCLinks(ctx context.Context, startDays int, endDays int) ([]IRCLink, error) {
···
293
293
err := s.db.WithContext(ctx).Raw(query, limit, offset).Scan(&results).Error
294
294
return results, err
295
295
}
296
+
297
+
func (s *GormStore) GetLinkPreview(ctx context.Context, url string) (*LinkPreview, error) {
298
+
var preview LinkPreview
299
+
err := s.db.WithContext(ctx).Where("url = ?", url).First(&preview).Error
300
+
if err != nil {
301
+
if err == gorm.ErrRecordNotFound {
302
+
return nil, nil
303
+
}
304
+
return nil, err
305
+
}
306
+
return &preview, nil
307
+
}
308
+
309
+
func (s *GormStore) InsertLinkPreview(ctx context.Context, url string, data []byte) error {
310
+
preview := LinkPreview{
311
+
URL: url,
312
+
Data: data,
313
+
}
314
+
// Use Save to handle upserts (update if exists) or explicit Replace
315
+
// Clause properties for Upsert
316
+
return s.db.WithContext(ctx).Clauses(clause.OnConflict{
317
+
Columns: []clause.Column{{Name: "url"}},
318
+
DoUpdates: clause.AssignmentColumns([]string{"data", "updated_at"}),
319
+
}).Create(&preview).Error
320
+
}
321
+
322
+
func (s *GormStore) DeleteLinkPreview(ctx context.Context, url string) error {
323
+
return s.db.WithContext(ctx).Delete(&LinkPreview{}, "url = ?", url).Error
324
+
}
+83
internal/data/preview_test.go
+83
internal/data/preview_test.go
···
1
+
package data
2
+
3
+
import (
4
+
"context"
5
+
"testing"
6
+
"time"
7
+
8
+
"gorm.io/driver/sqlite"
9
+
"gorm.io/gorm"
10
+
)
11
+
12
+
func TestLinkPreviewOperations(t *testing.T) {
13
+
// Setup in-memory SQLite DB
14
+
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
15
+
if err != nil {
16
+
t.Fatalf("Failed to open db: %v", err)
17
+
}
18
+
19
+
store := NewGormStore(db)
20
+
if err := store.Bootstrap(context.Background()); err != nil {
21
+
t.Fatalf("Failed to bootstrap db: %v", err)
22
+
}
23
+
24
+
ctx := context.Background()
25
+
url := "http://example.com/test"
26
+
data := []byte(`{"title":"Test Title"}`)
27
+
28
+
// Test 1: Get non-existent
29
+
preview, err := store.GetLinkPreview(ctx, url)
30
+
if err != nil {
31
+
t.Errorf("GetLinkPreview returned error for missing key: %v", err)
32
+
}
33
+
if preview != nil {
34
+
t.Errorf("GetLinkPreview should return nil for missing key, got %v", preview)
35
+
}
36
+
37
+
// Test 2: Insert and Get
38
+
if err := store.InsertLinkPreview(ctx, url, data); err != nil {
39
+
t.Fatalf("InsertLinkPreview failed: %v", err)
40
+
}
41
+
42
+
preview, err = store.GetLinkPreview(ctx, url)
43
+
if err != nil {
44
+
t.Fatalf("GetLinkPreview failed: %v", err)
45
+
}
46
+
if preview == nil {
47
+
t.Fatalf("GetLinkPreview returned nil after insert")
48
+
}
49
+
if preview.URL != url {
50
+
t.Errorf("Expected URL %s, got %s", url, preview.URL)
51
+
}
52
+
if string(preview.Data) != string(data) {
53
+
t.Errorf("Expected Data %s, got %s", data, preview.Data)
54
+
}
55
+
56
+
// Test 3: Update (Insert again)
57
+
newData := []byte(`{"title":"Updated Title"}`)
58
+
time.Sleep(10 * time.Millisecond) // Ensure timestamp update
59
+
if err := store.InsertLinkPreview(ctx, url, newData); err != nil {
60
+
t.Fatalf("InsertLinkPreview (update) failed: %v", err)
61
+
}
62
+
63
+
preview, err = store.GetLinkPreview(ctx, url)
64
+
if err != nil {
65
+
t.Fatalf("GetLinkPreview failed: %v", err)
66
+
}
67
+
if string(preview.Data) != string(newData) {
68
+
t.Errorf("Expected Data %s, got %s", newData, preview.Data)
69
+
}
70
+
71
+
// Test 4: Delete
72
+
if err := store.DeleteLinkPreview(ctx, url); err != nil {
73
+
t.Fatalf("DeleteLinkPreview failed: %v", err)
74
+
}
75
+
76
+
preview, err = store.GetLinkPreview(ctx, url)
77
+
if err != nil {
78
+
t.Errorf("GetLinkPreview returned error after delete: %v", err)
79
+
}
80
+
if preview != nil {
81
+
t.Errorf("GetLinkPreview should return nil after delete")
82
+
}
83
+
}
+8
internal/data/schema.mysql
+8
internal/data/schema.mysql
···
51
51
-- Record schema versions
52
52
INSERT IGNORE INTO `schema_version` (`version`, `description`) VALUES (1, 'Initial schema');
53
53
INSERT IGNORE INTO `schema_version` (`version`, `description`) VALUES (2, 'Added content_type to ircLink');
54
+
55
+
CREATE TABLE IF NOT EXISTS `link_previews` (
56
+
`url` varchar(768) NOT NULL,
57
+
`data` MEDIUMTEXT,
58
+
`created_at` timestamp NULL DEFAULT NULL,
59
+
`updated_at` timestamp NULL DEFAULT NULL,
60
+
PRIMARY KEY (`url`)
61
+
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+7
internal/data/schema.sqlite
+7
internal/data/schema.sqlite
+17
internal/data/store.go
+17
internal/data/store.go
···
63
63
MD5Sum string `json:"md5sum"` // For images
64
64
}
65
65
66
+
type LinkPreview struct {
67
+
URL string `json:"url" gorm:"column:url;primaryKey"`
68
+
Data []byte `json:"data" gorm:"column:data;type:text"` // JSON blob of the map[string]string metadata
69
+
CreatedAt time.Time `json:"created_at" gorm:"column:created_at"`
70
+
UpdatedAt time.Time `json:"updated_at" gorm:"column:updated_at"`
71
+
}
72
+
73
+
// TableName overrides the table name to `link_previews`
74
+
func (LinkPreview) TableName() string {
75
+
return "link_previews"
76
+
}
77
+
66
78
type Store interface {
67
79
GetRecentIRCLinks(ctx context.Context, days int, offsetDays int) ([]IRCLink, error)
68
80
GetRecentImages(ctx context.Context, days int, offsetDays int) ([]Image, error)
···
83
95
GetLinksByUser(ctx context.Context, user string, limit int, offset int) ([]IRCLink, error)
84
96
GetUserTimeline(ctx context.Context, user string, filterType string, limit int, offset int) ([]TimelineItem, error)
85
97
GetGlobalTimeline(ctx context.Context, limit int, offset int) ([]TimelineItem, error)
98
+
99
+
// Caching
100
+
GetLinkPreview(ctx context.Context, url string) (*LinkPreview, error)
101
+
InsertLinkPreview(ctx context.Context, url string, data []byte) error
102
+
DeleteLinkPreview(ctx context.Context, url string) error
86
103
87
104
Bootstrap(ctx context.Context) error
88
105
+28
internal/handler/cache.go
+28
internal/handler/cache.go
···
1
+
package handler
2
+
3
+
import (
4
+
"encoding/json"
5
+
"net/http"
6
+
)
7
+
8
+
// InvalidateCacheHandler allows manual invalidation of a cached preview
9
+
// Route: /api/caching/invalidate?url=...
10
+
func (h *Handler) InvalidateCacheHandler(w http.ResponseWriter, r *http.Request) {
11
+
urlParam := r.URL.Query().Get("url")
12
+
w.Header().Set("Content-Type", "application/json")
13
+
14
+
if urlParam == "" {
15
+
w.WriteHeader(http.StatusBadRequest)
16
+
json.NewEncoder(w).Encode(map[string]string{"error": "No URL provided"})
17
+
return
18
+
}
19
+
20
+
// Delete from DB
21
+
err := h.Store.DeleteLinkPreview(r.Context(), urlParam)
22
+
if err != nil {
23
+
h.ServerError(w, r, err)
24
+
return
25
+
}
26
+
27
+
json.NewEncoder(w).Encode(map[string]string{"status": "ok", "message": "Cache invalidated"})
28
+
}
+3
internal/handler/docs.go
+3
internal/handler/docs.go
···
26
26
"GitCommit": version.CommitHash,
27
27
"GitCommitURL": fmt.Sprintf("https://github.com/websages/tumble/commit/%s", version.CommitHash),
28
28
"Hot": h.getHotHTML(r.Context()),
29
+
// Potentially pass api docs specific data here if we had a dynamic docs page,
30
+
// but docs.html is currently static + swagger ui.
31
+
// If we want to mention the invalidated endpoint, we might need to modify docs.html or openapi.json
29
32
}
30
33
31
34
w.Header().Set("Content-Type", "text/html")
+29
-4
internal/handler/preview.go
+29
-4
internal/handler/preview.go
···
62
62
return
63
63
}
64
64
65
+
// Check Cache (if enabled)
66
+
if h.Config.Caching.Enabled {
67
+
if cached, err := h.Store.GetLinkPreview(r.Context(), urlParam); err == nil && cached != nil {
68
+
var meta map[string]string
69
+
if err := json.Unmarshal(cached.Data, &meta); err == nil {
70
+
// Client-side Caching Header (24h)
71
+
w.Header().Set("Cache-Control", "public, max-age=86400")
72
+
json.NewEncoder(w).Encode(meta)
73
+
return
74
+
}
75
+
}
76
+
}
77
+
65
78
// 0. Special Hybrid Handlers
66
79
if strings.Contains(urlParam, "reddit.com") {
67
80
meta, err := h.GetRedditPreview(urlParam)
···
105
118
106
119
// 1. Try OEmbed
107
120
if meta, err := h.tryOEmbed(urlParam); err == nil {
108
-
json.NewEncoder(w).Encode(meta)
121
+
h.cacheAndRespond(w, r, urlParam, meta)
109
122
return
110
123
}
111
124
112
125
// 2. Fallback to generic scraping
113
-
h.fetchOGScrape(w, urlParam)
126
+
h.fetchOGScrape(w, r, urlParam)
127
+
}
128
+
129
+
func (h *Handler) cacheAndRespond(w http.ResponseWriter, r *http.Request, urlParam string, meta map[string]string) {
130
+
// Cache if enabled
131
+
if h.Config.Caching.Enabled {
132
+
if data, err := json.Marshal(meta); err == nil {
133
+
h.Store.InsertLinkPreview(r.Context(), urlParam, data)
134
+
}
135
+
}
136
+
// Client-side Caching Header (24h)
137
+
w.Header().Set("Cache-Control", "public, max-age=86400")
138
+
json.NewEncoder(w).Encode(meta)
114
139
}
115
140
116
141
func (h *Handler) tryOEmbed(targetURL string) (map[string]string, error) {
···
196
221
return meta, nil
197
222
}
198
223
199
-
func (h *Handler) fetchOGScrape(w http.ResponseWriter, urlParam string) {
224
+
func (h *Handler) fetchOGScrape(w http.ResponseWriter, r *http.Request, urlParam string) {
200
225
// Default UA
201
226
ua := "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
202
227
···
223
248
return
224
249
}
225
250
226
-
json.NewEncoder(w).Encode(meta)
251
+
h.cacheAndRespond(w, r, urlParam, meta)
227
252
}
228
253
229
254
func (h *Handler) scrapeOpenGraph(targetURL, userAgent string) (map[string]string, error) {