(READ ONLY) Margin is an open annotation layer for the internet. Powered by the AT Protocol.
margin.at
extension
web
atproto
comments
optimizations
+111
-65
+6
backend/internal/db/db.go
+6
backend/internal/db/db.go
···
268
268
}
269
269
270
270
db.Exec(`CREATE INDEX IF NOT EXISTS idx_annotations_target_hash ON annotations(target_hash)`)
271
+
db.Exec(`CREATE INDEX IF NOT EXISTS idx_annotations_target_source ON annotations(target_source)`)
271
272
db.Exec(`CREATE INDEX IF NOT EXISTS idx_annotations_author_did ON annotations(author_did)`)
272
273
db.Exec(`CREATE INDEX IF NOT EXISTS idx_annotations_motivation ON annotations(motivation)`)
273
274
db.Exec(`CREATE INDEX IF NOT EXISTS idx_annotations_created_at ON annotations(created_at DESC)`)
···
287
288
)`)
288
289
db.Exec(`CREATE INDEX IF NOT EXISTS idx_highlights_target_hash ON highlights(target_hash)`)
289
290
db.Exec(`CREATE INDEX IF NOT EXISTS idx_highlights_author_did ON highlights(author_did)`)
291
+
db.Exec(`CREATE INDEX IF NOT EXISTS idx_highlights_created_at ON highlights(created_at DESC)`)
290
292
291
293
db.Exec(`CREATE TABLE IF NOT EXISTS bookmarks (
292
294
uri TEXT PRIMARY KEY,
···
302
304
)`)
303
305
db.Exec(`CREATE INDEX IF NOT EXISTS idx_bookmarks_source_hash ON bookmarks(source_hash)`)
304
306
db.Exec(`CREATE INDEX IF NOT EXISTS idx_bookmarks_author_did ON bookmarks(author_did)`)
307
+
db.Exec(`CREATE INDEX IF NOT EXISTS idx_bookmarks_created_at ON bookmarks(created_at DESC)`)
305
308
306
309
db.Exec(`CREATE TABLE IF NOT EXISTS replies (
307
310
uri TEXT PRIMARY KEY,
···
316
319
)`)
317
320
db.Exec(`CREATE INDEX IF NOT EXISTS idx_replies_parent_uri ON replies(parent_uri)`)
318
321
db.Exec(`CREATE INDEX IF NOT EXISTS idx_replies_root_uri ON replies(root_uri)`)
322
+
db.Exec(`CREATE INDEX IF NOT EXISTS idx_replies_created_at ON replies(created_at DESC)`)
319
323
320
324
db.Exec(`CREATE TABLE IF NOT EXISTS likes (
321
325
uri TEXT PRIMARY KEY,
···
338
342
indexed_at ` + dateType + ` NOT NULL
339
343
)`)
340
344
db.Exec(`CREATE INDEX IF NOT EXISTS idx_collections_author_did ON collections(author_did)`)
345
+
db.Exec(`CREATE INDEX IF NOT EXISTS idx_collections_created_at ON collections(created_at DESC)`)
341
346
342
347
db.Exec(`CREATE TABLE IF NOT EXISTS collection_items (
343
348
uri TEXT PRIMARY KEY,
···
350
355
)`)
351
356
db.Exec(`CREATE INDEX IF NOT EXISTS idx_collection_items_collection ON collection_items(collection_uri)`)
352
357
db.Exec(`CREATE INDEX IF NOT EXISTS idx_collection_items_annotation ON collection_items(annotation_uri)`)
358
+
db.Exec(`CREATE INDEX IF NOT EXISTS idx_collection_items_created_at ON collection_items(created_at DESC)`)
353
359
354
360
db.Exec(`CREATE TABLE IF NOT EXISTS sessions (
355
361
id TEXT PRIMARY KEY,
+25
-16
backend/internal/db/queries_annotations.go
+25
-16
backend/internal/db/queries_annotations.go
···
135
135
func (db *DB) GetPopularAnnotations(limit, offset int) ([]Annotation, error) {
136
136
since := time.Now().AddDate(0, 0, -14)
137
137
rows, err := db.Query(db.Rebind(`
138
-
SELECT uri, author_did, motivation, body_value, body_format, body_uri, target_source, target_hash, target_title, selector_json, tags_json, created_at, indexed_at, cid
139
-
FROM annotations
140
-
WHERE created_at > ? AND (
141
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = annotations.uri) +
142
-
(SELECT COUNT(*) FROM replies WHERE root_uri = annotations.uri)
143
-
) > 0
144
-
ORDER BY (
145
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = annotations.uri) +
146
-
(SELECT COUNT(*) FROM replies WHERE root_uri = annotations.uri)
147
-
) DESC, created_at DESC
138
+
SELECT
139
+
a.uri, a.author_did, a.motivation, a.body_value, a.body_format,
140
+
a.body_uri, a.target_source, a.target_hash, a.target_title,
141
+
a.selector_json, a.tags_json, a.created_at, a.indexed_at, a.cid
142
+
FROM annotations a
143
+
LEFT JOIN (
144
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
145
+
) l ON l.subject_uri = a.uri
146
+
LEFT JOIN (
147
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
148
+
) r ON r.root_uri = a.uri
149
+
WHERE a.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) > 0
150
+
ORDER BY (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) DESC, a.created_at DESC
148
151
LIMIT ? OFFSET ?
149
152
`), since, limit, offset)
150
153
if err != nil {
···
159
162
olderThan := time.Now().AddDate(0, 0, -1)
160
163
since := time.Now().AddDate(0, 0, -14)
161
164
rows, err := db.Query(db.Rebind(`
162
-
SELECT uri, author_did, motivation, body_value, body_format, body_uri, target_source, target_hash, target_title, selector_json, tags_json, created_at, indexed_at, cid
163
-
FROM annotations
164
-
WHERE created_at < ? AND created_at > ? AND (
165
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = annotations.uri) +
166
-
(SELECT COUNT(*) FROM replies WHERE root_uri = annotations.uri)
167
-
) = 0
165
+
SELECT
166
+
a.uri, a.author_did, a.motivation, a.body_value, a.body_format,
167
+
a.body_uri, a.target_source, a.target_hash, a.target_title,
168
+
a.selector_json, a.tags_json, a.created_at, a.indexed_at, a.cid
169
+
FROM annotations a
170
+
LEFT JOIN (
171
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
172
+
) l ON l.subject_uri = a.uri
173
+
LEFT JOIN (
174
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
175
+
) r ON r.root_uri = a.uri
176
+
WHERE a.created_at < ? AND a.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) = 0
168
177
ORDER BY RANDOM()
169
178
LIMIT ? OFFSET ?
170
179
`), olderThan, since, limit, offset)
+23
-16
backend/internal/db/queries_bookmarks.go
+23
-16
backend/internal/db/queries_bookmarks.go
···
59
59
func (db *DB) GetPopularBookmarks(limit, offset int) ([]Bookmark, error) {
60
60
since := time.Now().AddDate(0, 0, -14)
61
61
rows, err := db.Query(db.Rebind(`
62
-
SELECT uri, author_did, source, source_hash, title, description, tags_json, created_at, indexed_at, cid
63
-
FROM bookmarks
64
-
WHERE created_at > ? AND (
65
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = bookmarks.uri) +
66
-
(SELECT COUNT(*) FROM replies WHERE root_uri = bookmarks.uri)
67
-
) > 0
68
-
ORDER BY (
69
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = bookmarks.uri) +
70
-
(SELECT COUNT(*) FROM replies WHERE root_uri = bookmarks.uri)
71
-
) DESC, created_at DESC
62
+
SELECT
63
+
b.uri, b.author_did, b.source, b.source_hash, b.title,
64
+
b.description, b.tags_json, b.created_at, b.indexed_at, b.cid
65
+
FROM bookmarks b
66
+
LEFT JOIN (
67
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
68
+
) l ON l.subject_uri = b.uri
69
+
LEFT JOIN (
70
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
71
+
) r ON r.root_uri = b.uri
72
+
WHERE b.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) > 0
73
+
ORDER BY (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) DESC, b.created_at DESC
72
74
LIMIT ? OFFSET ?
73
75
`), since, limit, offset)
74
76
if err != nil {
···
91
93
olderThan := time.Now().AddDate(0, 0, -1)
92
94
since := time.Now().AddDate(0, 0, -14)
93
95
rows, err := db.Query(db.Rebind(`
94
-
SELECT uri, author_did, source, source_hash, title, description, tags_json, created_at, indexed_at, cid
95
-
FROM bookmarks
96
-
WHERE created_at < ? AND created_at > ? AND (
97
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = bookmarks.uri) +
98
-
(SELECT COUNT(*) FROM replies WHERE root_uri = bookmarks.uri)
99
-
) = 0
96
+
SELECT
97
+
b.uri, b.author_did, b.source, b.source_hash, b.title,
98
+
b.description, b.tags_json, b.created_at, b.indexed_at, b.cid
99
+
FROM bookmarks b
100
+
LEFT JOIN (
101
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
102
+
) l ON l.subject_uri = b.uri
103
+
LEFT JOIN (
104
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
105
+
) r ON r.root_uri = b.uri
106
+
WHERE b.created_at < ? AND b.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) = 0
100
107
ORDER BY RANDOM()
101
108
LIMIT ? OFFSET ?
102
109
`), olderThan, since, limit, offset)
+23
-16
backend/internal/db/queries_collections.go
+23
-16
backend/internal/db/queries_collections.go
···
123
123
func (db *DB) GetPopularCollectionItems(limit, offset int) ([]CollectionItem, error) {
124
124
since := time.Now().AddDate(0, 0, -14)
125
125
rows, err := db.Query(db.Rebind(`
126
-
SELECT uri, author_did, collection_uri, annotation_uri, position, created_at, indexed_at
127
-
FROM collection_items
128
-
WHERE created_at > ? AND (
129
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = collection_items.annotation_uri) +
130
-
(SELECT COUNT(*) FROM replies WHERE root_uri = collection_items.annotation_uri)
131
-
) > 0
132
-
ORDER BY (
133
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = collection_items.annotation_uri) +
134
-
(SELECT COUNT(*) FROM replies WHERE root_uri = collection_items.annotation_uri)
135
-
) DESC, created_at DESC
126
+
SELECT
127
+
c.uri, c.author_did, c.collection_uri, c.annotation_uri,
128
+
c.position, c.created_at, c.indexed_at
129
+
FROM collection_items c
130
+
LEFT JOIN (
131
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
132
+
) l ON l.subject_uri = c.annotation_uri
133
+
LEFT JOIN (
134
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
135
+
) r ON r.root_uri = c.annotation_uri
136
+
WHERE c.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) > 0
137
+
ORDER BY (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) DESC, c.created_at DESC
136
138
LIMIT ? OFFSET ?
137
139
`), since, limit, offset)
138
140
if err != nil {
···
155
157
olderThan := time.Now().AddDate(0, 0, -1)
156
158
since := time.Now().AddDate(0, 0, -14)
157
159
rows, err := db.Query(db.Rebind(`
158
-
SELECT uri, author_did, collection_uri, annotation_uri, position, created_at, indexed_at
159
-
FROM collection_items
160
-
WHERE created_at < ? AND created_at > ? AND (
161
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = collection_items.annotation_uri) +
162
-
(SELECT COUNT(*) FROM replies WHERE root_uri = collection_items.annotation_uri)
163
-
) = 0
160
+
SELECT
161
+
c.uri, c.author_did, c.collection_uri, c.annotation_uri,
162
+
c.position, c.created_at, c.indexed_at
163
+
FROM collection_items c
164
+
LEFT JOIN (
165
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
166
+
) l ON l.subject_uri = c.annotation_uri
167
+
LEFT JOIN (
168
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
169
+
) r ON r.root_uri = c.annotation_uri
170
+
WHERE c.created_at < ? AND c.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) = 0
164
171
ORDER BY RANDOM()
165
172
LIMIT ? OFFSET ?
166
173
`), olderThan, since, limit, offset)
+23
-16
backend/internal/db/queries_highlights.go
+23
-16
backend/internal/db/queries_highlights.go
···
60
60
func (db *DB) GetPopularHighlights(limit, offset int) ([]Highlight, error) {
61
61
since := time.Now().AddDate(0, 0, -14)
62
62
rows, err := db.Query(db.Rebind(`
63
-
SELECT uri, author_did, target_source, target_hash, target_title, selector_json, color, tags_json, created_at, indexed_at, cid
64
-
FROM highlights
65
-
WHERE created_at > ? AND (
66
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = highlights.uri) +
67
-
(SELECT COUNT(*) FROM replies WHERE root_uri = highlights.uri)
68
-
) > 0
69
-
ORDER BY (
70
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = highlights.uri) +
71
-
(SELECT COUNT(*) FROM replies WHERE root_uri = highlights.uri)
72
-
) DESC, created_at DESC
63
+
SELECT
64
+
h.uri, h.author_did, h.target_source, h.target_hash, h.target_title,
65
+
h.selector_json, h.color, h.tags_json, h.created_at, h.indexed_at, h.cid
66
+
FROM highlights h
67
+
LEFT JOIN (
68
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
69
+
) l ON l.subject_uri = h.uri
70
+
LEFT JOIN (
71
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
72
+
) r ON r.root_uri = h.uri
73
+
WHERE h.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) > 0
74
+
ORDER BY (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) DESC, h.created_at DESC
73
75
LIMIT ? OFFSET ?
74
76
`), since, limit, offset)
75
77
if err != nil {
···
92
94
olderThan := time.Now().AddDate(0, 0, -1)
93
95
since := time.Now().AddDate(0, 0, -14)
94
96
rows, err := db.Query(db.Rebind(`
95
-
SELECT uri, author_did, target_source, target_hash, target_title, selector_json, color, tags_json, created_at, indexed_at, cid
96
-
FROM highlights
97
-
WHERE created_at < ? AND created_at > ? AND (
98
-
(SELECT COUNT(*) FROM likes WHERE subject_uri = highlights.uri) +
99
-
(SELECT COUNT(*) FROM replies WHERE root_uri = highlights.uri)
100
-
) = 0
97
+
SELECT
98
+
h.uri, h.author_did, h.target_source, h.target_hash, h.target_title,
99
+
h.selector_json, h.color, h.tags_json, h.created_at, h.indexed_at, h.cid
100
+
FROM highlights h
101
+
LEFT JOIN (
102
+
SELECT subject_uri, COUNT(*) as cnt FROM likes GROUP BY subject_uri
103
+
) l ON l.subject_uri = h.uri
104
+
LEFT JOIN (
105
+
SELECT root_uri, COUNT(*) as cnt FROM replies GROUP BY root_uri
106
+
) r ON r.root_uri = h.uri
107
+
WHERE h.created_at < ? AND h.created_at > ? AND (COALESCE(l.cnt, 0) + COALESCE(r.cnt, 0)) = 0
101
108
ORDER BY RANDOM()
102
109
LIMIT ? OFFSET ?
103
110
`), olderThan, since, limit, offset)
+11
-1
backend/internal/db/queries_recommendations.go
+11
-1
backend/internal/db/queries_recommendations.go
···
422
422
}
423
423
424
424
func (db *DB) GetCandidateDocuments(userDID string, limit int) ([]CandidateDocument, error) {
425
+
// Note: We use NOT LIKE instead of !~* for cross-database compatibility and performance.
426
+
// The engagement count sub-select is also constrained to recent elements if possible, but
427
+
// for now we just optimize the regex and exact grouping.
425
428
rows, err := db.Query(db.Rebind(`
426
429
SELECT
427
430
d.uri, d.author_did, d.site, d.path, d.title, d.description, d.tags_json,
···
440
443
AND (p.show_in_discover IS NULL OR p.show_in_discover = true)
441
444
AND LENGTH(d.title) > 15
442
445
AND (LENGTH(COALESCE(d.description, '')) >= 30 OR LENGTH(COALESCE(d.text_content, '')) >= 100)
443
-
AND d.title !~* '(^test$|^test\\s|\\stest$|^testing|^hello\\sworld|^untitled|^draft|^asdf|^lorem|^foo$|^bar$|^placeholder)'
446
+
AND LOWER(d.title) NOT LIKE '%test%'
447
+
AND LOWER(d.title) NOT LIKE '%testing%'
448
+
AND LOWER(d.title) NOT LIKE '%hello world%'
449
+
AND LOWER(d.title) NOT LIKE '%untitled%'
450
+
AND LOWER(d.title) NOT LIKE '%draft%'
451
+
AND LOWER(d.title) NOT LIKE '%asdf%'
452
+
AND LOWER(d.title) NOT LIKE '%lorem%'
453
+
AND LOWER(d.title) NOT LIKE '%placeholder%'
444
454
AND d.uri NOT IN (
445
455
SELECT DISTINCT document_uri FROM annotation_embeddings
446
456
WHERE author_did = ? AND document_uri IS NOT NULL