···1111 "pkg.rbrt.fr/glean/internal/db"
1212)
13131414+const (
1515+ actionCreate = "create"
1616+ actionUpdate = "update"
1717+ actionDelete = "delete"
1818+)
1919+1420var sentinelErrors = []error{db.ErrDuplicateSubscription, db.ErrDuplicateLike}
15211622func isSentinel(err error) bool {
···52585359func (h *StreamDBHandler) handleSubscription(ctx context.Context, event *Event) error {
5460 switch event.Type {
5555- case "create", "update":
6161+ case actionCreate, actionUpdate:
5662 var rec SubscriptionRecord
5763 if err := json.Unmarshal(event.Value, &rec); err != nil {
5864 return err
···6874 }
6975 return err
70767171- case "delete":
7777+ case actionDelete:
7278 parsed, ok := ParseRecordURI(event.URI)
7379 if !ok {
7480 return nil
···84908591func (h *StreamDBHandler) handleLike(ctx context.Context, event *Event) error {
8692 switch event.Type {
8787- case "create":
9393+ case actionCreate:
8894 var rec LikeRecord
8995 if err := json.Unmarshal(event.Value, &rec); err != nil {
9096 return err
···107113 }
108114 return err
109115110110- case "delete":
116116+ case actionDelete:
111117 return h.articles.DeleteLike(ctx, event.URI)
112118 }
113119 return nil
···115121116122func (h *StreamDBHandler) handleAnnotation(ctx context.Context, event *Event) error {
117123 switch event.Type {
118118- case "create":
124124+ case actionCreate:
119125 var rec AnnotationRecord
120126 if err := json.Unmarshal(event.Value, &rec); err != nil {
121127 return err
···141147 }
142148 return h.articles.CreateAnnotation(ctx, a)
143149144144- case "delete":
150150+ case actionDelete:
145151 return h.articles.DeleteAnnotation(ctx, event.URI)
146152 }
147153 return nil
···149155150156func (h *StreamDBHandler) handleFollow(ctx context.Context, event *Event) error {
151157 switch event.Type {
152152- case "create":
158158+ case actionCreate:
153159 var rec FollowRecord
154160 if err := json.Unmarshal(event.Value, &rec); err != nil {
155161 return err
···159165 }
160166 return h.users.UpsertFollow(ctx, event.DID, rec.Subject, event.URI, event.CID)
161167162162- case "delete":
168168+ case actionDelete:
163169 return h.users.DeleteFollowByURI(ctx, event.URI)
164170 }
165171 return nil
···167173168174func (h *StreamDBHandler) handleMarginNote(ctx context.Context, event *Event) error {
169175 switch event.Type {
170170- case "create", "update":
176176+ case actionCreate, actionUpdate:
171177 var rec MarginNoteRecord
172178 if err := json.Unmarshal(event.Value, &rec); err != nil {
173179 return err
···194200 }
195201 return h.articles.CreateAnnotation(ctx, a)
196202197197- case "delete":
203203+ case actionDelete:
198204 }
199205 return nil
200206}
201207202208func (h *StreamDBHandler) handleSkyreaderSubscription(ctx context.Context, event *Event) error {
203209 switch event.Type {
204204- case "create", "update":
210210+ case actionCreate, actionUpdate:
205211 var rec SkyreaderSubscriptionRecord
206212 if err := json.Unmarshal(event.Value, &rec); err != nil {
207213 return err
···217223 }
218224 return err
219225220220- case "delete":
226226+ case actionDelete:
221227 }
222228 return nil
223229}
+2-2
internal/atproto/xrpc.go
···430430 var did, feedURL, title string
431431 var cat sql.NullString
432432 if err := subRows.Scan(&did, &feedURL, &title, &cat); err != nil {
433433- subRows.Close()
433433+ _ = subRows.Close()
434434 http.Error(w, err.Error(), http.StatusInternalServerError)
435435 return
436436 }
···440440 Category: cat.String,
441441 })
442442 }
443443- subRows.Close()
443443+ _ = subRows.Close()
444444 }
445445446446 for _, u := range users {
+6-5
internal/db/article.go
···1010 "pkg.rbrt.fr/glean/internal/feed"
1111)
12121313+const articlesOrderBy = ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
1414+1315type ArticleStore struct {
1416 db *DB
1517}
···142144 }
143145144146 // Future-published articles (e.g., scheduled) sort last
145145- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
147147+ query += articlesOrderBy
146148 args = append(args, limit, offset)
147149148150 rows, err := s.db.QueryContext(ctx, query, args...)
···206208 }
207209208210 // Future-published articles (e.g., scheduled) sort last
209209- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
211211+ query += articlesOrderBy
210212 args = append(args, limit, offset)
211213212214 rows, err := s.db.QueryContext(ctx, query, args...)
···270272 }
271273272274 // Future-published articles (e.g., scheduled) sort last
273273- query += ` ORDER BY (CASE WHEN a.published > 'now' THEN 1 ELSE 0 END), a.published DESC LIMIT ? OFFSET ?`
275275+ query += articlesOrderBy
274276 args = append(args, limit, offset)
275277276278 rows, err := s.db.QueryContext(ctx, query, args...)
···339341func (s *ArticleStore) GetReadState(ctx context.Context, userDID string, articleID int64) (*ReadState, error) {
340342 rs := &ReadState{}
341343 err := s.db.QueryRowContext(ctx, `
342342- SELECT user_did, article_id, is_read, read_at
343343- FROM articles.read_state WHERE user_did = ? AND article_id = ?
344344+ SELECT * FROM articles.read_state WHERE user_did = ? AND article_id = ?
344345 `, userDID, articleID).Scan(&rs.UserDID, &rs.ArticleID, &rs.IsRead, &rs.ReadAt)
345346 if err == sql.ErrNoRows {
346347 return &ReadState{UserDID: userDID, ArticleID: articleID}, nil
+319-11
internal/db/db.go
···2233import (
44 "database/sql"
55+ "fmt"
56 "math"
67 "strings"
88+ "sync/atomic"
79 "time"
810911 "github.com/mattn/go-sqlite3"
···11131214const DSN = "_journal_mode=WAL&_busy_timeout=30000&_synchronous=NORMAL&_cache=shared"
13151414-func init() {
1515- sql.Register("sqlite3_glean", &sqlite3.SQLiteDriver{
1616+type DB struct {
1717+ *sql.DB
1818+}
1919+2020+type Databases struct {
2121+ Users *UserStore
2222+ Articles *ArticleStore
2323+2424+ db *DB
2525+}
2626+2727+var multiDriverSeq int64
2828+2929+func OpenAll(basePath string) (*Databases, error) {
3030+ articlesPath := basePath + "_articles"
3131+ recsPath := basePath + "_recs"
3232+3333+ seq := atomic.AddInt64(&multiDriverSeq, 1)
3434+ driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq)
3535+3636+ sql.Register(driverName, &sqlite3.SQLiteDriver{
1637 ConnectHook: func(conn *sqlite3.SQLiteConn) error {
1717- if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil {
3838+ if err := conn.RegisterFunc("exp", math.Exp, true); err != nil {
1839 return err
1940 }
2020- if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil {
4141+ if err := conn.RegisterFunc("log", math.Log, true); err != nil {
2142 return err
2243 }
2323- pragmas := []string{
4444+ for _, p := range []string{
2445 `PRAGMA wal_autocheckpoint = 1000`,
2546 `PRAGMA temp_store = MEMORY`,
2647 `PRAGMA mmap_size = 268435456`,
2727- }
2828- for _, p := range pragmas {
4848+ } {
2949 if _, err := conn.Exec(p, nil); err != nil {
3050 return err
3151 }
3252 }
5353+ if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil {
5454+ return err
5555+ }
5656+ if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil {
5757+ return err
5858+ }
3359 return nil
3460 },
3561 })
6262+6363+ db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN)
6464+ if err != nil {
6565+ return nil, err
6666+ }
6767+ db.SetMaxOpenConns(10)
6868+ db.SetMaxIdleConns(5)
6969+ db.SetConnMaxLifetime(30 * time.Minute)
7070+ d := &DB{db}
7171+7272+ if err := initUsersSchema(d); err != nil {
7373+ d.Close()
7474+ return nil, err
7575+ }
7676+7777+ if err := initArticlesSchema(d); err != nil {
7878+ d.Close()
7979+ return nil, err
8080+ }
8181+8282+ if err := initRecsSchema(d); err != nil {
8383+ d.Close()
8484+ return nil, err
8585+ }
8686+8787+ return &Databases{
8888+ Users: NewUserStore(d),
8989+ Articles: NewArticleStore(d),
9090+ db: d,
9191+ }, nil
9292+}
9393+9494+func (d *Databases) Close() error {
9595+ if d.db != nil {
9696+ _ = d.db.Close()
9797+ }
9898+ return nil
9999+}
100100+101101+func (d *Databases) DB() *sql.DB {
102102+ return d.db.DB
103103+}
104104+105105+func initUsersSchema(db *DB) error {
106106+ for _, s := range usersSchema {
107107+ if _, err := db.Exec(s); err != nil {
108108+ return err
109109+ }
110110+ }
111111+ return nil
112112+}
113113+114114+func initArticlesSchema(db *DB) error {
115115+ for _, s := range articlesSchema {
116116+ if _, err := db.Exec(s); err != nil {
117117+ return err
118118+ }
119119+ }
120120+ return nil
121121+}
122122+123123+func initRecsSchema(db *DB) error {
124124+ for _, s := range recsSchema {
125125+ if _, err := db.Exec(s); err != nil {
126126+ return err
127127+ }
128128+ }
129129+ return nil
130130+}
131131+132132+var usersSchema = []string{
133133+ `CREATE TABLE IF NOT EXISTS users (
134134+ did TEXT PRIMARY KEY,
135135+ indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
136136+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
137137+ )`,
138138+139139+ `CREATE TABLE IF NOT EXISTS follows (
140140+ user_did TEXT NOT NULL,
141141+ target_did TEXT NOT NULL,
142142+ uri TEXT,
143143+ cid TEXT,
144144+ followed_at DATETIME,
145145+ PRIMARY KEY (user_did, target_did)
146146+ )`,
147147+148148+ `CREATE TABLE IF NOT EXISTS oauth_auth_requests (
149149+ state TEXT PRIMARY KEY,
150150+ data TEXT NOT NULL
151151+ )`,
152152+153153+ `CREATE TABLE IF NOT EXISTS oauth_sessions (
154154+ account_did TEXT NOT NULL,
155155+ session_id TEXT NOT NULL,
156156+ data TEXT NOT NULL,
157157+ PRIMARY KEY (account_did, session_id)
158158+ )`,
159159+160160+ `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
161161+ `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
162162+ `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
163163+ `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
164164+}
165165+166166+var articlesSchema = []string{
167167+ `CREATE TABLE IF NOT EXISTS articles.feeds (
168168+ feed_url TEXT PRIMARY KEY,
169169+ title TEXT,
170170+ site_url TEXT,
171171+ description TEXT,
172172+ feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
173173+ last_fetched_at DATETIME,
174174+ last_error TEXT,
175175+ subscriber_count INTEGER NOT NULL DEFAULT 0,
176176+ etag TEXT,
177177+ last_modified TEXT,
178178+ consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
179179+ error_count INTEGER NOT NULL DEFAULT 0,
180180+ favicon_url TEXT
181181+ )`,
182182+183183+ `CREATE TABLE IF NOT EXISTS articles.subscriptions (
184184+ id INTEGER PRIMARY KEY AUTOINCREMENT,
185185+ user_did TEXT NOT NULL,
186186+ feed_url TEXT NOT NULL,
187187+ title TEXT,
188188+ category TEXT,
189189+ added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
190190+ uri TEXT,
191191+ cid TEXT,
192192+ UNIQUE(user_did, feed_url)
193193+ )`,
194194+195195+ `CREATE TABLE IF NOT EXISTS articles.articles (
196196+ id INTEGER PRIMARY KEY AUTOINCREMENT,
197197+ feed_url TEXT NOT NULL,
198198+ guid TEXT NOT NULL,
199199+ title TEXT NOT NULL DEFAULT '',
200200+ url TEXT,
201201+ author TEXT,
202202+ summary TEXT,
203203+ content TEXT,
204204+ full_content TEXT,
205205+ published DATETIME,
206206+ updated DATETIME,
207207+ fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
208208+ UNIQUE(feed_url, guid)
209209+ )`,
210210+211211+ `CREATE TABLE IF NOT EXISTS articles.read_state (
212212+ user_did TEXT NOT NULL,
213213+ article_id INTEGER NOT NULL,
214214+ is_read BOOLEAN NOT NULL DEFAULT 0,
215215+ read_at DATETIME,
216216+ PRIMARY KEY (user_did, article_id)
217217+ )`,
218218+219219+ `CREATE TABLE IF NOT EXISTS articles.annotations (
220220+ id INTEGER PRIMARY KEY AUTOINCREMENT,
221221+ uri TEXT NOT NULL UNIQUE,
222222+ author_did TEXT NOT NULL,
223223+ feed_url TEXT NOT NULL,
224224+ article_url TEXT NOT NULL,
225225+ quote TEXT,
226226+ note TEXT,
227227+ tags TEXT,
228228+ rating INTEGER,
229229+ created_at DATETIME NOT NULL,
230230+ cid TEXT
231231+ )`,
232232+233233+ `CREATE TABLE IF NOT EXISTS articles.likes (
234234+ id INTEGER PRIMARY KEY AUTOINCREMENT,
235235+ uri TEXT NOT NULL UNIQUE,
236236+ author_did TEXT NOT NULL,
237237+ feed_url TEXT NOT NULL,
238238+ article_url TEXT NOT NULL,
239239+ created_at DATETIME NOT NULL,
240240+ cid TEXT,
241241+ UNIQUE(author_did, feed_url, article_url)
242242+ )`,
243243+244244+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
245245+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
246246+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
247247+ `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
248248+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
249249+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
250250+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
251251+ `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
252252+ `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
253253+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
254254+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
255255+ `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
256256+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
257257+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
258258+ `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
259259+260260+ `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
261261+ `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
262262+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
263263+ END`,
264264+ `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
265265+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
266266+ END`,
267267+ `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
268268+ INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
269269+ INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
270270+ END`,
271271+}
272272+273273+var recsSchema = []string{
274274+ `CREATE TABLE IF NOT EXISTS recs.feed_similarity (
275275+ feed_a TEXT NOT NULL,
276276+ feed_b TEXT NOT NULL,
277277+ jaccard REAL NOT NULL,
278278+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
279279+ PRIMARY KEY (feed_a, feed_b),
280280+ CHECK(feed_a < feed_b)
281281+ )`,
282282+283283+ `CREATE TABLE IF NOT EXISTS recs.user_similarity (
284284+ user_a TEXT NOT NULL,
285285+ user_b TEXT NOT NULL,
286286+ jaccard REAL NOT NULL,
287287+ common_feeds INTEGER NOT NULL,
288288+ common_likes INTEGER NOT NULL DEFAULT 0,
289289+ common_tags INTEGER NOT NULL DEFAULT 0,
290290+ computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
291291+ PRIMARY KEY (user_a, user_b),
292292+ CHECK(user_a < user_b)
293293+ )`,
294294+295295+ `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations (
296296+ user_did TEXT NOT NULL,
297297+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
298298+ target_id TEXT NOT NULL,
299299+ reason TEXT,
300300+ dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
301301+ PRIMARY KEY (user_did, target_type, target_id)
302302+ )`,
303303+304304+ `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions (
305305+ user_did TEXT NOT NULL,
306306+ target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
307307+ target_id TEXT NOT NULL,
308308+ first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
309309+ last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
310310+ shown_count INTEGER NOT NULL DEFAULT 1,
311311+ acted BOOLEAN NOT NULL DEFAULT 0,
312312+ PRIMARY KEY (user_did, target_type, target_id)
313313+ )`,
314314+315315+ `CREATE TABLE IF NOT EXISTS recs.follow_distances (
316316+ user_a TEXT NOT NULL,
317317+ user_b TEXT NOT NULL,
318318+ distance INTEGER NOT NULL CHECK(distance IN (1, 2)),
319319+ PRIMARY KEY (user_a, user_b)
320320+ )`,
321321+322322+ `CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
323323+ user_did TEXT PRIMARY KEY,
324324+ w_sub REAL NOT NULL DEFAULT 1.0,
325325+ w_like REAL NOT NULL DEFAULT 0.5,
326326+ w_tag REAL NOT NULL DEFAULT 0.3,
327327+ w_social REAL NOT NULL DEFAULT 0.7,
328328+ w_pop REAL NOT NULL DEFAULT 0.2,
329329+ w_category REAL NOT NULL DEFAULT 0.4,
330330+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
331331+ )`,
332332+333333+ `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
334334+ user_did TEXT PRIMARY KEY,
335335+ total_likes INTEGER NOT NULL DEFAULT 0,
336336+ total_tags INTEGER NOT NULL DEFAULT 0,
337337+ top_categories TEXT,
338338+ updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
339339+ )`,
340340+341341+ `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
342342+ `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
343343+ `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
344344+ `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
345345+ `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
346346+ `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
347347+ `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
36348}
3734938350func NullStr(s string) sql.NullString {
···53365 }
54366 return sql.NullString{String: strings.Join(tags, ","), Valid: true}
55367}
5656-5757-type DB struct {
5858- *sql.DB
5959-}
+5-8
internal/db/feed.go
···5454 FaviconURL sql.NullString
5555}
56565757-const feedSelectCols = `feed_url, title, site_url, description, feed_type,
5858- last_fetched_at, last_error, subscriber_count, etag, last_modified,
5959- consecutive_empty_fetches, error_count, favicon_url`
60576158func scanFeed(scanner interface{ Scan(...any) error }) (*Feed, error) {
6259 f := &Feed{}
···7370}
74717572func (s *ArticleStore) GetFeed(ctx context.Context, feedURL string) (*Feed, error) {
7676- row := s.db.QueryRowContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds WHERE feed_url = ?`, feedURL)
7373+ row := s.db.QueryRowContext(ctx, `SELECT * FROM articles.feeds WHERE feed_url = ?`, feedURL)
7774 return scanFeed(row)
7875}
79768077func (s *ArticleStore) GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error) {
8178 cutoff := time.Now().Add(-olderThan)
8282- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds
7979+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
8380 WHERE subscriber_count > 0 AND error_count < 25 AND (last_fetched_at IS NULL OR last_fetched_at <= ?)
8481 ORDER BY last_fetched_at ASC NULLS FIRST LIMIT ?`, cutoff, limit)
8582 if err != nil {
···309306}
310307311308func (s *ArticleStore) ListDeadFeeds(ctx context.Context, userDID string, threshold int) ([]*Feed, error) {
312312- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds f
309309+ rows, err := s.db.QueryContext(ctx, `SELECT f.* FROM articles.feeds f
313310 JOIN articles.subscriptions s ON s.feed_url = f.feed_url AND s.user_did = ?
314311 WHERE f.error_count >= ? ORDER BY f.error_count DESC`, userDID, threshold)
315312 if err != nil {
···329326}
330327331328func (s *ArticleStore) ListAllFeeds(ctx context.Context, limit, offset int) ([]*Feed, error) {
332332- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds
329329+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
333330 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, limit, offset)
334331 if err != nil {
335332 return nil, err
···459456}
460457461458func (s *ArticleStore) ListUnsubscribedFeeds(ctx context.Context, userDID string, limit, offset int) ([]*Feed, error) {
462462- rows, err := s.db.QueryContext(ctx, `SELECT `+feedSelectCols+` FROM articles.feeds
459459+ rows, err := s.db.QueryContext(ctx, `SELECT * FROM articles.feeds
463460 WHERE feed_url NOT IN (SELECT feed_url FROM articles.subscriptions WHERE user_did = ?)
464461 ORDER BY subscriber_count DESC LIMIT ? OFFSET ?`, userDID, limit, offset)
465462 if err != nil {
+1-2
internal/db/follow.go
···60606161func (s *UserStore) ListFollowers(ctx context.Context, targetDID string, limit, offset int) ([]*Follow, error) {
6262 rows, err := s.db.QueryContext(ctx, `
6363- SELECT user_did, target_did, uri, cid, followed_at
6464- FROM follows WHERE target_did = ?
6363+ SELECT * FROM follows WHERE target_did = ?
6564 ORDER BY followed_at DESC
6665 LIMIT ? OFFSET ?
6766 `, targetDID, limit, offset)
-341
internal/db/multi.go
···11-package db
22-33-import (
44- "database/sql"
55- "fmt"
66- "math"
77- "sync/atomic"
88- "time"
99-1010- "github.com/mattn/go-sqlite3"
1111-)
1212-1313-type Databases struct {
1414- Users *UserStore
1515- Articles *ArticleStore
1616-1717- db *DB
1818-}
1919-2020-var multiDriverSeq int64
2121-2222-func OpenAll(basePath string) (*Databases, error) {
2323- articlesPath := basePath + "_articles"
2424- recsPath := basePath + "_recs"
2525-2626- seq := atomic.AddInt64(&multiDriverSeq, 1)
2727- driverName := fmt.Sprintf("sqlite3_glean_multi_%d", seq)
2828-2929- sql.Register(driverName, &sqlite3.SQLiteDriver{
3030- ConnectHook: func(conn *sqlite3.SQLiteConn) error {
3131- if err := conn.RegisterFunc("exp", func(x float64) float64 { return math.Exp(x) }, true); err != nil {
3232- return err
3333- }
3434- if err := conn.RegisterFunc("log", func(x float64) float64 { return math.Log(x) }, true); err != nil {
3535- return err
3636- }
3737- for _, p := range []string{
3838- `PRAGMA wal_autocheckpoint = 1000`,
3939- `PRAGMA temp_store = MEMORY`,
4040- `PRAGMA mmap_size = 268435456`,
4141- } {
4242- if _, err := conn.Exec(p, nil); err != nil {
4343- return err
4444- }
4545- }
4646- if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS articles", articlesPath), nil); err != nil {
4747- return err
4848- }
4949- if _, err := conn.Exec(fmt.Sprintf("ATTACH DATABASE '%s' AS recs", recsPath), nil); err != nil {
5050- return err
5151- }
5252- return nil
5353- },
5454- })
5555-5656- db, err := sql.Open(driverName, basePath+"_users?cache=shared&"+DSN)
5757- if err != nil {
5858- return nil, err
5959- }
6060- db.SetMaxOpenConns(10)
6161- db.SetMaxIdleConns(5)
6262- db.SetConnMaxLifetime(30 * time.Minute)
6363- d := &DB{db}
6464-6565- if err := initUsersSchema(d); err != nil {
6666- d.Close()
6767- return nil, err
6868- }
6969-7070- if err := initArticlesSchema(d); err != nil {
7171- d.Close()
7272- return nil, err
7373- }
7474-7575- if err := initRecsSchema(d); err != nil {
7676- d.Close()
7777- return nil, err
7878- }
7979-8080- return &Databases{
8181- Users: NewUserStore(d),
8282- Articles: NewArticleStore(d),
8383- db: d,
8484- }, nil
8585-}
8686-8787-func (d *Databases) Close() error {
8888- if d.db != nil {
8989- _ = d.db.Close()
9090- }
9191- return nil
9292-}
9393-9494-func (d *Databases) DB() *sql.DB {
9595- return d.db.DB
9696-}
9797-9898-func initUsersSchema(db *DB) error {
9999- for _, s := range usersSchema {
100100- if _, err := db.Exec(s); err != nil {
101101- return err
102102- }
103103- }
104104- return nil
105105-}
106106-107107-func initArticlesSchema(db *DB) error {
108108- for _, s := range articlesSchema {
109109- if _, err := db.Exec(s); err != nil {
110110- return err
111111- }
112112- }
113113- return nil
114114-}
115115-116116-func initRecsSchema(db *DB) error {
117117- for _, s := range recsSchema {
118118- if _, err := db.Exec(s); err != nil {
119119- return err
120120- }
121121- }
122122- return nil
123123-}
124124-125125-var usersSchema = []string{
126126- `CREATE TABLE IF NOT EXISTS users (
127127- did TEXT PRIMARY KEY,
128128- indexed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
129129- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
130130- )`,
131131-132132- `CREATE TABLE IF NOT EXISTS follows (
133133- user_did TEXT NOT NULL,
134134- target_did TEXT NOT NULL,
135135- uri TEXT,
136136- cid TEXT,
137137- followed_at DATETIME,
138138- PRIMARY KEY (user_did, target_did)
139139- )`,
140140-141141- `CREATE TABLE IF NOT EXISTS oauth_auth_requests (
142142- state TEXT PRIMARY KEY,
143143- data TEXT NOT NULL
144144- )`,
145145-146146- `CREATE TABLE IF NOT EXISTS oauth_sessions (
147147- account_did TEXT NOT NULL,
148148- session_id TEXT NOT NULL,
149149- data TEXT NOT NULL,
150150- PRIMARY KEY (account_did, session_id)
151151- )`,
152152-153153- `CREATE INDEX IF NOT EXISTS idx_follows_user ON follows(user_did)`,
154154- `CREATE INDEX IF NOT EXISTS idx_follows_target ON follows(target_did)`,
155155- `CREATE INDEX IF NOT EXISTS idx_follows_uri ON follows(uri)`,
156156- `CREATE INDEX IF NOT EXISTS idx_follows_followed_at ON follows(followed_at)`,
157157-}
158158-159159-var articlesSchema = []string{
160160- `CREATE TABLE IF NOT EXISTS articles.feeds (
161161- feed_url TEXT PRIMARY KEY,
162162- title TEXT,
163163- site_url TEXT,
164164- description TEXT,
165165- feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')),
166166- last_fetched_at DATETIME,
167167- last_error TEXT,
168168- subscriber_count INTEGER NOT NULL DEFAULT 0,
169169- etag TEXT,
170170- last_modified TEXT,
171171- consecutive_empty_fetches INTEGER NOT NULL DEFAULT 0,
172172- error_count INTEGER NOT NULL DEFAULT 0,
173173- favicon_url TEXT
174174- )`,
175175-176176- `CREATE TABLE IF NOT EXISTS articles.subscriptions (
177177- id INTEGER PRIMARY KEY AUTOINCREMENT,
178178- user_did TEXT NOT NULL,
179179- feed_url TEXT NOT NULL,
180180- title TEXT,
181181- category TEXT,
182182- added_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
183183- uri TEXT,
184184- cid TEXT,
185185- UNIQUE(user_did, feed_url)
186186- )`,
187187-188188- `CREATE TABLE IF NOT EXISTS articles.articles (
189189- id INTEGER PRIMARY KEY AUTOINCREMENT,
190190- feed_url TEXT NOT NULL,
191191- guid TEXT NOT NULL,
192192- title TEXT NOT NULL DEFAULT '',
193193- url TEXT,
194194- author TEXT,
195195- summary TEXT,
196196- content TEXT,
197197- full_content TEXT,
198198- published DATETIME,
199199- updated DATETIME,
200200- fetched_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
201201- UNIQUE(feed_url, guid)
202202- )`,
203203-204204- `CREATE TABLE IF NOT EXISTS articles.read_state (
205205- user_did TEXT NOT NULL,
206206- article_id INTEGER NOT NULL,
207207- is_read BOOLEAN NOT NULL DEFAULT 0,
208208- read_at DATETIME,
209209- PRIMARY KEY (user_did, article_id)
210210- )`,
211211-212212- `CREATE TABLE IF NOT EXISTS articles.annotations (
213213- id INTEGER PRIMARY KEY AUTOINCREMENT,
214214- uri TEXT NOT NULL UNIQUE,
215215- author_did TEXT NOT NULL,
216216- feed_url TEXT NOT NULL,
217217- article_url TEXT NOT NULL,
218218- quote TEXT,
219219- note TEXT,
220220- tags TEXT,
221221- rating INTEGER,
222222- created_at DATETIME NOT NULL,
223223- cid TEXT
224224- )`,
225225-226226- `CREATE TABLE IF NOT EXISTS articles.likes (
227227- id INTEGER PRIMARY KEY AUTOINCREMENT,
228228- uri TEXT NOT NULL UNIQUE,
229229- author_did TEXT NOT NULL,
230230- feed_url TEXT NOT NULL,
231231- article_url TEXT NOT NULL,
232232- created_at DATETIME NOT NULL,
233233- cid TEXT,
234234- UNIQUE(author_did, feed_url, article_url)
235235- )`,
236236-237237- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed ON subscriptions(feed_url)`,
238238- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_feed_user ON subscriptions(feed_url, user_did)`,
239239- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_user ON subscriptions(user_did)`,
240240- `CREATE INDEX IF NOT EXISTS articles.idx_subscriptions_uri ON subscriptions(uri)`,
241241- `CREATE INDEX IF NOT EXISTS articles.idx_likes_author_feed ON likes(author_did, feed_url, created_at)`,
242242- `CREATE INDEX IF NOT EXISTS articles.idx_articles_feed ON articles(feed_url)`,
243243- `CREATE INDEX IF NOT EXISTS articles.idx_articles_published ON articles(published DESC)`,
244244- `CREATE INDEX IF NOT EXISTS articles.idx_articles_url ON articles(url)`,
245245- `CREATE INDEX IF NOT EXISTS articles.idx_read_state_unread ON read_state(user_did, is_read) WHERE is_read = 0`,
246246- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_article ON annotations(article_url)`,
247247- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_author ON annotations(author_did)`,
248248- `CREATE INDEX IF NOT EXISTS articles.idx_annotations_created_at ON annotations(created_at DESC)`,
249249- `CREATE INDEX IF NOT EXISTS articles.idx_likes_article ON likes(feed_url, article_url)`,
250250- `CREATE INDEX IF NOT EXISTS articles.idx_likes_author ON likes(author_did)`,
251251- `CREATE INDEX IF NOT EXISTS articles.idx_likes_created_at ON likes(created_at DESC)`,
252252-253253- `CREATE VIRTUAL TABLE IF NOT EXISTS articles.articles_fts USING fts5(title, summary, content, author, content=articles, content_rowid=id)`,
254254- `CREATE TRIGGER IF NOT EXISTS articles.articles_ai AFTER INSERT ON articles BEGIN
255255- INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
256256- END`,
257257- `CREATE TRIGGER IF NOT EXISTS articles.articles_ad AFTER DELETE ON articles BEGIN
258258- INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
259259- END`,
260260- `CREATE TRIGGER IF NOT EXISTS articles.articles_au AFTER UPDATE ON articles BEGIN
261261- INSERT INTO articles_fts(articles_fts, rowid, title, summary, content, author) VALUES('delete', old.id, old.title, old.summary, old.content, old.author);
262262- INSERT INTO articles_fts(rowid, title, summary, content, author) VALUES (new.id, new.title, new.summary, new.content, new.author);
263263- END`,
264264-}
265265-266266-var recsSchema = []string{
267267- `CREATE TABLE IF NOT EXISTS recs.feed_similarity (
268268- feed_a TEXT NOT NULL,
269269- feed_b TEXT NOT NULL,
270270- jaccard REAL NOT NULL,
271271- computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
272272- PRIMARY KEY (feed_a, feed_b),
273273- CHECK(feed_a < feed_b)
274274- )`,
275275-276276- `CREATE TABLE IF NOT EXISTS recs.user_similarity (
277277- user_a TEXT NOT NULL,
278278- user_b TEXT NOT NULL,
279279- jaccard REAL NOT NULL,
280280- common_feeds INTEGER NOT NULL,
281281- common_likes INTEGER NOT NULL DEFAULT 0,
282282- common_tags INTEGER NOT NULL DEFAULT 0,
283283- computed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
284284- PRIMARY KEY (user_a, user_b),
285285- CHECK(user_a < user_b)
286286- )`,
287287-288288- `CREATE TABLE IF NOT EXISTS recs.dismissed_recommendations (
289289- user_did TEXT NOT NULL,
290290- target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
291291- target_id TEXT NOT NULL,
292292- reason TEXT,
293293- dismissed_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
294294- PRIMARY KEY (user_did, target_type, target_id)
295295- )`,
296296-297297- `CREATE TABLE IF NOT EXISTS recs.recommendation_impressions (
298298- user_did TEXT NOT NULL,
299299- target_type TEXT NOT NULL CHECK(target_type IN ('feed', 'article')),
300300- target_id TEXT NOT NULL,
301301- first_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
302302- last_shown_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
303303- shown_count INTEGER NOT NULL DEFAULT 1,
304304- acted BOOLEAN NOT NULL DEFAULT 0,
305305- PRIMARY KEY (user_did, target_type, target_id)
306306- )`,
307307-308308- `CREATE TABLE IF NOT EXISTS recs.follow_distances (
309309- user_a TEXT NOT NULL,
310310- user_b TEXT NOT NULL,
311311- distance INTEGER NOT NULL CHECK(distance IN (1, 2)),
312312- PRIMARY KEY (user_a, user_b)
313313- )`,
314314-315315- `CREATE TABLE IF NOT EXISTS recs.user_signal_weights (
316316- user_did TEXT PRIMARY KEY,
317317- w_sub REAL NOT NULL DEFAULT 1.0,
318318- w_like REAL NOT NULL DEFAULT 0.5,
319319- w_tag REAL NOT NULL DEFAULT 0.3,
320320- w_social REAL NOT NULL DEFAULT 0.7,
321321- w_pop REAL NOT NULL DEFAULT 0.2,
322322- w_category REAL NOT NULL DEFAULT 0.4,
323323- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
324324- )`,
325325-326326- `CREATE TABLE IF NOT EXISTS recs.user_signal_profiles (
327327- user_did TEXT PRIMARY KEY,
328328- total_likes INTEGER NOT NULL DEFAULT 0,
329329- total_tags INTEGER NOT NULL DEFAULT 0,
330330- top_categories TEXT,
331331- updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
332332- )`,
333333-334334- `CREATE INDEX IF NOT EXISTS recs.idx_dismissed_user_type ON dismissed_recommendations(user_did, target_type)`,
335335- `CREATE INDEX IF NOT EXISTS recs.idx_impressions_user_unacted ON recommendation_impressions(user_did, acted, shown_count)`,
336336- `CREATE INDEX IF NOT EXISTS recs.idx_impressions_last_shown ON recommendation_impressions(last_shown_at)`,
337337- `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_b ON follow_distances(user_b)`,
338338- `CREATE INDEX IF NOT EXISTS recs.idx_follow_distances_a_dist ON follow_distances(user_a, distance)`,
339339- `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_b ON user_similarity(user_b)`,
340340- `CREATE INDEX IF NOT EXISTS recs.idx_user_similarity_a ON user_similarity(user_a)`,
341341-}
+1-1
internal/db/social.go
···240240func (s *ArticleStore) GetLike(ctx context.Context, authorDID, feedURL, articleURL string) (*Like, error) {
241241 l := &Like{}
242242 err := s.db.QueryRowContext(ctx, `
243243- SELECT id, uri, author_did, feed_url, article_url, created_at, cid FROM articles.likes
243243+ SELECT * FROM articles.likes
244244 WHERE author_did = ? AND feed_url = ? AND article_url = ?
245245 `, authorDID, feedURL, articleURL).Scan(&l.ID, &l.URI, &l.AuthorDID, &l.FeedURL, &l.ArticleURL, &l.CreatedAt, &l.CID)
246246 if err != nil {
+1-2
internal/db/user.go
···6060func (s *UserStore) GetUser(ctx context.Context, did string) (*User, error) {
6161 u := &User{}
6262 err := s.db.QueryRowContext(ctx, `
6363- SELECT did, indexed_at, updated_at
6464- FROM users WHERE did = ?
6363+ SELECT * FROM users WHERE did = ?
6564 `, did).Scan(&u.DID, &u.IndexedAt, &u.UpdatedAt)
6665 if err != nil {
6766 return nil, err