A social RSS reader built on the AT Protocol.
glean.at
glean
atproto
atmosphere
rss
feed
social
app
Optimize database queries with indexes and temp tables
+78
-24
+6
internal/cluster/jaccard.go
+6
internal/cluster/jaccard.go
···
133
133
return err
134
134
}
135
135
136
+
if _, err := tx.ExecContext(ctx, `
137
+
CREATE INDEX IF NOT EXISTS _idx_feed_words_word ON _feed_words(word, feed_url)
138
+
`); err != nil {
139
+
return err
140
+
}
141
+
136
142
descInsert := `
137
143
INSERT OR IGNORE INTO feed_similarity (feed_a, feed_b, jaccard)
138
144
SELECT feed_a, feed_b, 0 FROM _word_overlap
+64
-12
internal/cluster/scoring.go
+64
-12
internal/cluster/scoring.go
···
286
286
return err
287
287
}
288
288
289
+
if _, err := tx.ExecContext(ctx, `
290
+
CREATE TEMP TABLE IF NOT EXISTS _user_like_counts (user_did TEXT PRIMARY KEY, cnt INT)
291
+
`); err != nil {
292
+
return err
293
+
}
294
+
if _, err := tx.ExecContext(ctx, `DELETE FROM _user_like_counts`); err != nil {
295
+
return err
296
+
}
297
+
if _, err := tx.ExecContext(ctx, `
298
+
INSERT INTO _user_like_counts SELECT author_did, COUNT(*) FROM likes GROUP BY author_did
299
+
`); err != nil {
300
+
return err
301
+
}
302
+
303
+
if _, err := tx.ExecContext(ctx, `
304
+
CREATE TEMP TABLE IF NOT EXISTS _user_tag_counts (user_did TEXT PRIMARY KEY, cnt INT)
305
+
`); err != nil {
306
+
return err
307
+
}
308
+
if _, err := tx.ExecContext(ctx, `DELETE FROM _user_tag_counts`); err != nil {
309
+
return err
310
+
}
311
+
if _, err := tx.ExecContext(ctx, `
312
+
INSERT INTO _user_tag_counts
313
+
WITH user_tags AS (
314
+
SELECT author_did, TRIM(value) AS tag
315
+
FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
316
+
WHERE tags IS NOT NULL AND tags != ''
317
+
)
318
+
SELECT author_did, COUNT(DISTINCT tag) FROM user_tags GROUP BY author_did
319
+
`); err != nil {
320
+
return err
321
+
}
322
+
323
+
if _, err := tx.ExecContext(ctx, `
324
+
CREATE TEMP TABLE IF NOT EXISTS _user_top_categories (user_did TEXT PRIMARY KEY, categories TEXT)
325
+
`); err != nil {
326
+
return err
327
+
}
328
+
if _, err := tx.ExecContext(ctx, `DELETE FROM _user_top_categories`); err != nil {
329
+
return err
330
+
}
331
+
if _, err := tx.ExecContext(ctx, `
332
+
INSERT INTO _user_top_categories
333
+
SELECT user_did, '[' || GROUP_CONCAT('{"c":"' || category || '","n":"' || CAST(cnt AS TEXT) || '}') || ']'
334
+
FROM (
335
+
SELECT user_did, category, COUNT(*) AS cnt
336
+
FROM subscriptions
337
+
WHERE category IS NOT NULL AND category != ''
338
+
GROUP BY user_did, category
339
+
ORDER BY COUNT(*) DESC
340
+
LIMIT 5
341
+
)
342
+
GROUP BY user_did
343
+
`); err != nil {
344
+
return err
345
+
}
346
+
289
347
_, err = tx.ExecContext(ctx, `
290
348
INSERT INTO user_signal_profiles (user_did, total_likes, total_tags, top_categories)
291
349
SELECT
292
350
u.did,
293
-
(SELECT COUNT(*) FROM likes WHERE author_did = u.did),
294
-
COALESCE((SELECT COUNT(DISTINCT TRIM(value))
295
-
FROM annotations, json_each('["' || REPLACE(tags, ',', '","') || '"]')
296
-
WHERE author_did = u.did AND tags IS NOT NULL AND tags != ''
297
-
), 0),
298
-
(SELECT '[' || GROUP_CONCAT('{"c":"' || category || '","n":"' || CAST(cnt AS TEXT) || '}') || ']'
299
-
FROM (
300
-
SELECT category, COUNT(*) AS cnt
301
-
FROM subscriptions WHERE user_did = u.did AND category IS NOT NULL AND category != ''
302
-
GROUP BY category ORDER BY cnt DESC LIMIT 5
303
-
)
304
-
)
351
+
COALESCE(lc.cnt, 0),
352
+
COALESCE(tc.cnt, 0),
353
+
COALESCE(cc.categories, '[]')
305
354
FROM users u
355
+
LEFT JOIN _user_like_counts lc ON lc.user_did = u.did
356
+
LEFT JOIN _user_tag_counts tc ON tc.user_did = u.did
357
+
LEFT JOIN _user_top_categories cc ON cc.user_did = u.did
306
358
`)
307
359
if err != nil {
308
360
return err
+3
-1
internal/db/db.go
+3
-1
internal/db/db.go
···
207
207
PRIMARY KEY (account_did, session_id)
208
208
)`,
209
209
`CREATE INDEX IF NOT EXISTS idx_subscriptions_feed ON subscriptions(feed_url)`,
210
+
`CREATE INDEX IF NOT EXISTS idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
210
211
`CREATE INDEX IF NOT EXISTS idx_subscriptions_user ON subscriptions(user_did)`,
211
212
`CREATE INDEX IF NOT EXISTS idx_subscriptions_uri ON subscriptions(uri)`,
213
+
`CREATE INDEX IF NOT EXISTS idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
212
214
`CREATE INDEX IF NOT EXISTS idx_articles_feed ON articles(feed_url)`,
213
215
`CREATE INDEX IF NOT EXISTS idx_articles_published ON articles(published DESC)`,
214
216
`CREATE INDEX IF NOT EXISTS idx_articles_url ON articles(url)`,
···
223
225
`CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
224
226
`CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
225
227
`CREATE INDEX IF NOT EXISTS idx_user_similarity_b ON user_similarity(user_b)`,
228
+
`CREATE INDEX IF NOT EXISTS idx_user_similarity_a ON user_similarity(user_a)`,
226
229
227
230
`CREATE TABLE IF NOT EXISTS dismissed_recommendations (
228
231
user_did TEXT NOT NULL REFERENCES users(did),
···
275
278
`CREATE INDEX IF NOT EXISTS idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
276
279
`CREATE INDEX IF NOT EXISTS idx_follow_distances_b ON follow_distances(user_b)`,
277
280
`CREATE INDEX IF NOT EXISTS idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
278
-
`CREATE INDEX IF NOT EXISTS idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
279
281
`CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
280
282
`CREATE INDEX IF NOT EXISTS idx_users_handle ON users(handle)`,
281
283
`CREATE VIRTUAL TABLE IF NOT EXISTS articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,