A social RSS reader built on the AT Protocol.
glean.at
glean
atproto
atmosphere
rss
feed
social
app
Improve fetcher performance and cleanup endpoints
+102
-52
-1
docs/specs.md
-1
docs/specs.md
···
844
844
| `/dashboard` | GET | Main dashboard: unread articles, recommendations sidebar |
845
845
| `/feeds` | GET | Manage RSS subscriptions (OPML import for onboarding) |
846
846
| `/feeds/list` | GET | Feed list fragment (htmx partial) |
847
-
| `/feeds/discover-url` | GET | Discover feed URL from a website |
848
847
| `/feeds/opml/upload` | POST | Upload OPML file to bulk-import subscriptions |
849
848
| `/feeds/opml/download` | GET | Export subscriptions as OPML (offboarding) |
850
849
| `/feeds/add` | POST | Add a single feed URL |
+1
-1
internal/cluster/scoring.go
+1
-1
internal/cluster/scoring.go
···
224
224
JOIN articles a ON a.feed_url = la.feed_url AND a.url = la.article_url
225
225
LEFT JOIN feeds f ON f.feed_url = la.feed_url
226
226
LEFT JOIN social_likes sl ON sl.feed_url = la.feed_url AND sl.article_url = la.article_url
227
-
ORDER BY score DESC
227
+
ORDER BY score DESC, a.published DESC
228
228
LIMIT ?
229
229
`, userDID, userDID, userDID, userDID, userDID, userDID, w.WLike, w.WSocial, limit)
230
230
if err != nil {
+52
internal/db/article.go
+52
internal/db/article.go
···
6
6
"strings"
7
7
"time"
8
8
"unicode"
9
+
10
+
"pkg.rbrt.fr/glean/internal/feed"
9
11
)
10
12
11
13
type Article struct {
···
50
52
`, article.FeedURL, article.GUID).Scan(&id)
51
53
}
52
54
return id, err
55
+
}
56
+
57
+
func (db *DB) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
58
+
if len(articles) == 0 {
59
+
return nil
60
+
}
61
+
62
+
err := upsertArticlesBatch(ctx, db, articles)
63
+
if err != nil {
64
+
err = upsertArticlesBatch(ctx, db, articles)
65
+
}
66
+
return err
67
+
}
68
+
69
+
func upsertArticlesBatch(ctx context.Context, db *DB, articles []feed.Article) error {
70
+
tx, err := db.BeginTx(ctx, nil)
71
+
if err != nil {
72
+
return err
73
+
}
74
+
defer tx.Rollback()
75
+
76
+
stmt, err := tx.PrepareContext(ctx, `
77
+
INSERT INTO articles (feed_url, guid, title, url, author, summary, content, published, updated)
78
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
79
+
ON CONFLICT(feed_url, guid) DO NOTHING
80
+
`)
81
+
if err != nil {
82
+
return err
83
+
}
84
+
defer stmt.Close()
85
+
86
+
for _, a := range articles {
87
+
url := sql.NullString{String: a.URL, Valid: a.URL != ""}
88
+
author := sql.NullString{String: a.Author, Valid: a.Author != ""}
89
+
summary := sql.NullString{String: a.Summary, Valid: a.Summary != ""}
90
+
content := sql.NullString{String: a.Content, Valid: a.Content != ""}
91
+
var published, updated sql.NullTime
92
+
if !a.Published.IsZero() {
93
+
published = sql.NullTime{Time: a.Published, Valid: true}
94
+
}
95
+
if !a.Updated.IsZero() {
96
+
updated = sql.NullTime{Time: a.Updated, Valid: true}
97
+
}
98
+
99
+
if _, err := stmt.ExecContext(ctx, a.FeedURL, a.GUID, a.Title, url, author, summary, content, published, updated); err != nil {
100
+
return err
101
+
}
102
+
}
103
+
104
+
return tx.Commit()
53
105
}
54
106
55
107
func (db *DB) GetArticle(ctx context.Context, id int64) (*Article, error) {
+4
internal/db/store.go
+4
internal/db/store.go
···
56
56
return a.db.UpsertArticle(ctx, dbArticle)
57
57
}
58
58
59
+
func (a *FeedStoreAdapter) UpsertArticlesBatch(ctx context.Context, articles []feed.Article) error {
60
+
return a.db.UpsertArticlesBatch(ctx, articles)
61
+
}
62
+
59
63
func (a *FeedStoreAdapter) MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error {
60
64
return a.db.MarkFeedFetched(ctx, feedURL, etag, lastModified)
61
65
}
+34
-13
internal/feed/discover.go
+34
-13
internal/feed/discover.go
···
116
116
origin.RawQuery = ""
117
117
origin.Fragment = ""
118
118
119
+
type result struct {
120
+
url string
121
+
found bool
122
+
}
123
+
found := make(chan result, 1)
124
+
ctx, cancel := context.WithCancel(ctx)
125
+
defer cancel()
126
+
119
127
for _, path := range faviconPaths {
120
-
u, _ := url.Parse(path)
121
-
resolved := origin.ResolveReference(u)
122
-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, resolved.String(), nil)
123
-
if err != nil {
124
-
continue
128
+
go func(path string) {
129
+
u, _ := url.Parse(path)
130
+
resolved := origin.ResolveReference(u)
131
+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, resolved.String(), nil)
132
+
if err != nil {
133
+
return
134
+
}
135
+
resp, err := discoverClient.Do(req)
136
+
if err != nil {
137
+
return
138
+
}
139
+
resp.Body.Close()
140
+
if resp.StatusCode == http.StatusOK && imageContentTypes.matches(resp.Header.Get("Content-Type")) {
141
+
select {
142
+
case found <- result{url: cleanFavicon(resolved.String()), found: true}:
143
+
default:
144
+
}
145
+
}
146
+
}(path)
147
+
}
148
+
149
+
select {
150
+
case r := <-found:
151
+
if r.found {
152
+
return r.url
125
153
}
126
-
resp, err := discoverClient.Do(req)
127
-
if err != nil {
128
-
continue
129
-
}
130
-
resp.Body.Close()
131
-
if resp.StatusCode == http.StatusOK && imageContentTypes.matches(resp.Header.Get("Content-Type")) {
132
-
return cleanFavicon(resolved.String())
133
-
}
154
+
case <-ctx.Done():
134
155
}
135
156
return ""
136
157
}
+9
-8
internal/feed/fetcher.go
+9
-8
internal/feed/fetcher.go
···
70
70
type FeedStore interface {
71
71
GetFeedsToFetch(ctx context.Context, olderThan time.Duration, limit int) ([]*Feed, error)
72
72
UpsertArticle(ctx context.Context, article *Article) (int64, error)
73
+
UpsertArticlesBatch(ctx context.Context, articles []Article) error
73
74
MarkFeedFetched(ctx context.Context, feedURL, etag, lastModified string) error
74
75
MarkFeedFetchError(ctx context.Context, feedURL, lastError string) error
75
76
UpdateFeedFavicon(ctx context.Context, feedURL, faviconURL string) error
···
120
121
return
121
122
}
122
123
123
-
sem := make(chan struct{}, 3)
124
+
sem := make(chan struct{}, 10)
124
125
var wg sync.WaitGroup
125
126
for _, f := range feeds {
126
127
wg.Add(1)
···
169
170
170
171
metrics.FeedsFetched.WithLabelValues("success").Inc()
171
172
172
-
for i := range result.Articles {
173
-
result.Articles[i].FeedURL = feed.URL
174
-
if _, upsertErr := s.store.UpsertArticle(ctx, &result.Articles[i]); upsertErr != nil {
175
-
s.logger.Error("failed to upsert article", "error", upsertErr, "url", result.Articles[i].URL)
176
-
} else {
177
-
metrics.ArticlesUpserted.Inc()
178
-
}
173
+
for _, article := range result.Articles {
174
+
article.FeedURL = feed.URL
175
+
}
176
+
if err := s.store.UpsertArticlesBatch(ctx, result.Articles); err != nil {
177
+
s.logger.Error("failed to upsert articles", "error", err, "feed", feed.URL)
178
+
} else {
179
+
metrics.ArticlesUpserted.Add(float64(len(result.Articles)))
179
180
}
180
181
181
182
if err := s.store.MarkFeedFetched(ctx, feed.URL, newEtag, newLastModified); err != nil {
-26
internal/server/feeds_handler.go
-26
internal/server/feeds_handler.go
···
3
3
import (
4
4
"context"
5
5
"database/sql"
6
-
"encoding/json"
7
6
"errors"
8
7
"net/http"
9
8
"time"
···
371
370
372
371
s.render(w, r, "dead-feeds.html", map[string]any{
373
372
"DeadFeeds": deadFeeds,
374
-
})
375
-
}
376
-
377
-
func (s *Server) handleDiscoverFeedURL(w http.ResponseWriter, r *http.Request) {
378
-
siteURL := r.URL.Query().Get("url")
379
-
if siteURL == "" {
380
-
http.Error(w, "url required", http.StatusBadRequest)
381
-
return
382
-
}
383
-
384
-
result, err := feed.Discover(r.Context(), siteURL)
385
-
if err != nil {
386
-
s.logger.Error("feed discovery failed", "error", err, "url", siteURL)
387
-
http.Error(w, err.Error(), http.StatusInternalServerError)
388
-
return
389
-
}
390
-
391
-
w.Header().Set("Content-Type", "application/json")
392
-
type discoveryResponse struct {
393
-
FeedURLs []string `json:"feed_urls"`
394
-
Favicon string `json:"favicon"`
395
-
}
396
-
json.NewEncoder(w).Encode(discoveryResponse{
397
-
FeedURLs: result.FeedURLs,
398
-
Favicon: result.Favicon,
399
373
})
400
374
}
401
375
-1
internal/server/server.go
-1
internal/server/server.go
···
161
161
r.Post("/refresh", s.handleRefreshFeeds)
162
162
r.Post("/retry", s.handleRetryFeed)
163
163
r.Get("/list", s.handleFeedList)
164
-
r.Get("/discover-url", s.handleDiscoverFeedURL)
165
164
r.Post("/clear", s.handleClearAllSubscriptions)
166
165
r.Post("/dismiss", s.handleDismissFeedRecommendation)
167
166
})