A social RSS reader built on the AT Protocol.
glean.at
glean
atproto
atmosphere
rss
feed
social
app
Improve profile fetching
+272
-185
+16
internal/atproto/client.go
+16
internal/atproto/client.go
···
99
99
}
100
100
return records, result.Cursor, nil
101
101
}
102
+
103
+
func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) {
104
+
nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile")
105
+
if err != nil {
106
+
return "", "", fmt.Errorf("parsing NSID: %w", err)
107
+
}
108
+
109
+
var profile struct {
110
+
DisplayName string `json:"displayName"`
111
+
Avatar string `json:"avatar"`
112
+
}
113
+
if err := c.api.Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil {
114
+
return "", "", err
115
+
}
116
+
return profile.DisplayName, profile.Avatar, nil
117
+
}
+45
internal/atproto/profile.go
+45
internal/atproto/profile.go
···
1
+
package atproto
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"fmt"
7
+
"net/http"
8
+
"net/url"
9
+
"time"
10
+
)
11
+
12
+
var profileClient = &http.Client{Timeout: 10 * time.Second}
13
+
14
+
// FetchProfile is an unauthenticated profile fetcher. It relies on bluesky api.
15
+
func FetchProfile(ctx context.Context, identifier string) (handle, displayName, avatarURL string, err error) {
16
+
apiURL := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=%s", url.QueryEscape(identifier))
17
+
return fetchProfileFromURL(ctx, apiURL, identifier)
18
+
}
19
+
20
+
func fetchProfileFromURL(ctx context.Context, apiURL, identifier string) (handle, displayName, avatarURL string, err error) {
21
+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
22
+
if err != nil {
23
+
return "", "", "", err
24
+
}
25
+
26
+
resp, err := profileClient.Do(req)
27
+
if err != nil {
28
+
return "", "", "", err
29
+
}
30
+
defer resp.Body.Close()
31
+
32
+
if resp.StatusCode != http.StatusOK {
33
+
return "", "", "", fmt.Errorf("fetch profile %s: status %d", identifier, resp.StatusCode)
34
+
}
35
+
36
+
var profile struct {
37
+
Handle string `json:"handle"`
38
+
DisplayName string `json:"displayName"`
39
+
Avatar string `json:"avatar"`
40
+
}
41
+
if err := json.NewDecoder(resp.Body).Decode(&profile); err != nil {
42
+
return "", "", "", err
43
+
}
44
+
return profile.Handle, profile.DisplayName, profile.Avatar, nil
45
+
}
+66
internal/atproto/profile_test.go
+66
internal/atproto/profile_test.go
···
1
+
package atproto
2
+
3
+
import (
4
+
"context"
5
+
"encoding/json"
6
+
"net/http"
7
+
"net/http/httptest"
8
+
"net/url"
9
+
"testing"
10
+
11
+
"gotest.tools/v3/assert"
12
+
)
13
+
14
+
func TestFetchProfile_Success(t *testing.T) {
15
+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
16
+
assert.Equal(t, r.URL.Query().Get("actor"), "did:plc:test123")
17
+
json.NewEncoder(w).Encode(map[string]string{
18
+
"handle": "test.bsky.social",
19
+
"displayName": "Test User",
20
+
"avatar": "https://cdn.bsky.app/img/avatar/test.png",
21
+
})
22
+
}))
23
+
defer srv.Close()
24
+
25
+
origClient := profileClient
26
+
profileClient = srv.Client()
27
+
defer func() { profileClient = origClient }()
28
+
29
+
apiURL := srv.URL + "/xrpc/app.bsky.actor.getProfile?actor=" + url.QueryEscape("did:plc:test123")
30
+
handle, dn, avatar, err := fetchProfileFromURL(context.Background(), apiURL, "did:plc:test123")
31
+
assert.NilError(t, err)
32
+
assert.Equal(t, handle, "test.bsky.social")
33
+
assert.Equal(t, dn, "Test User")
34
+
assert.Equal(t, avatar, "https://cdn.bsky.app/img/avatar/test.png")
35
+
}
36
+
37
+
func TestFetchProfile_EmptyProfile(t *testing.T) {
38
+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
39
+
json.NewEncoder(w).Encode(map[string]string{})
40
+
}))
41
+
defer srv.Close()
42
+
43
+
origClient := profileClient
44
+
profileClient = srv.Client()
45
+
defer func() { profileClient = origClient }()
46
+
47
+
handle, dn, avatar, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123")
48
+
assert.NilError(t, err)
49
+
assert.Equal(t, handle, "")
50
+
assert.Equal(t, dn, "")
51
+
assert.Equal(t, avatar, "")
52
+
}
53
+
54
+
func TestFetchProfile_Non200(t *testing.T) {
55
+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
56
+
w.WriteHeader(http.StatusNotFound)
57
+
}))
58
+
defer srv.Close()
59
+
60
+
origClient := profileClient
61
+
profileClient = srv.Client()
62
+
defer func() { profileClient = origClient }()
63
+
64
+
_, _, _, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123")
65
+
assert.Assert(t, err != nil)
66
+
}
+8
-3
internal/atproto/sync.go
+8
-3
internal/atproto/sync.go
···
266
266
FollowedAt: db.NullTime(t),
267
267
}
268
268
269
-
var handle string
270
-
if rec.Subject != userDID {
271
-
s.db.CreateUser(ctx, rec.Subject, handle, "", "")
269
+
// auto onboard followers
270
+
var handle, displayName, avatarURL string
271
+
if h, dn, avatar, err := FetchProfile(ctx, rec.Subject); err == nil {
272
+
handle = h
273
+
displayName = dn
274
+
avatarURL = avatar
272
275
}
276
+
277
+
s.db.CreateUser(ctx, rec.Subject, handle, displayName, avatarURL)
273
278
}
274
279
275
280
if next == "" || len(records) == 0 {
+11
internal/db/user.go
+11
internal/db/user.go
···
72
72
return dids, rows.Err()
73
73
}
74
74
75
+
func (db *DB) UpdateUserProfile(ctx context.Context, did, displayName, avatarURL string) error {
76
+
_, err := db.ExecContext(ctx, `
77
+
UPDATE users SET
78
+
display_name = COALESCE(NULLIF(?, ''), display_name),
79
+
avatar_url = COALESCE(NULLIF(?, ''), avatar_url),
80
+
updated_at = CURRENT_TIMESTAMP
81
+
WHERE did = ?
82
+
`, displayName, avatarURL, did)
83
+
return err
84
+
}
85
+
75
86
func (db *DB) ListUsers(ctx context.Context) ([]*User, error) {
76
87
rows, err := db.QueryContext(ctx, `
77
88
SELECT did, handle, display_name, avatar_url, indexed_at, updated_at
+41
internal/db/user_test.go
+41
internal/db/user_test.go
···
1
+
package db
2
+
3
+
import (
4
+
"context"
5
+
"testing"
6
+
7
+
"gotest.tools/v3/assert"
8
+
)
9
+
10
+
func TestUpdateUserProfile_SetsFields(t *testing.T) {
11
+
ctx := context.Background()
12
+
db := setupTestDB(t)
13
+
14
+
_, err := db.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:profile", "tester")
15
+
assert.NilError(t, err)
16
+
17
+
err = db.UpdateUserProfile(ctx, "did:test:profile", "Display Name", "https://cdn.bsky.app/img/avatar.png")
18
+
assert.NilError(t, err)
19
+
20
+
u, err := db.GetUser(ctx, "did:test:profile")
21
+
assert.NilError(t, err)
22
+
assert.Equal(t, u.DisplayName.String, "Display Name")
23
+
assert.Equal(t, u.AvatarURL.String, "https://cdn.bsky.app/img/avatar.png")
24
+
}
25
+
26
+
func TestUpdateUserProfile_DoesNotOverwriteWithEmpty(t *testing.T) {
27
+
ctx := context.Background()
28
+
db := setupTestDB(t)
29
+
30
+
_, err := db.ExecContext(ctx, `INSERT INTO users (did, handle, display_name, avatar_url) VALUES (?, ?, ?, ?)`,
31
+
"did:test:profile2", "tester2", "Existing Name", "https://old.avatar/url")
32
+
assert.NilError(t, err)
33
+
34
+
err = db.UpdateUserProfile(ctx, "did:test:profile2", "", "")
35
+
assert.NilError(t, err)
36
+
37
+
u, err := db.GetUser(ctx, "did:test:profile2")
38
+
assert.NilError(t, err)
39
+
assert.Equal(t, u.DisplayName.String, "Existing Name")
40
+
assert.Equal(t, u.AvatarURL.String, "https://old.avatar/url")
41
+
}
+63
-126
internal/feed/discover.go
+63
-126
internal/feed/discover.go
···
2
2
3
3
import (
4
4
"context"
5
-
"fmt"
6
5
"io"
7
6
"net/http"
7
+
"net/url"
8
8
"regexp"
9
-
"strings"
10
9
"time"
11
10
)
12
11
···
16
15
}
17
16
18
17
var (
19
-
linkRe = regexp.MustCompile(`<link[^>]+>`)
20
-
hrefRe = regexp.MustCompile(`href="([^"]*)"`)
21
-
relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`)
22
-
typeFeedRe = regexp.MustCompile(`type="([^"]*(?:rss|atom|feed|xml)[^"]*)"`)
23
-
relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`)
24
-
baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`)
25
-
faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"}
18
+
linkRe = regexp.MustCompile(`<link[^>]+>`)
19
+
hrefRe = regexp.MustCompile(`href="([^"]*)"`)
20
+
relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`)
21
+
typeFeedRe = regexp.MustCompile(`type="[^"]*(?:rss|atom|feed|xml)[^"]*"`)
22
+
relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`)
23
+
baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`)
26
24
25
+
faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"}
27
26
discoverClient = &http.Client{Timeout: 15 * time.Second}
28
27
)
29
28
30
29
func Discover(ctx context.Context, siteURL string) (*DiscoveryResult, error) {
31
-
result := &DiscoveryResult{}
32
-
33
-
favicon := discoverFavicon(ctx, discoverClient, siteURL)
34
-
if favicon != "" {
35
-
result.Favicon = favicon
30
+
base, html := fetchHTML(ctx, siteURL)
31
+
if base == nil {
32
+
return &DiscoveryResult{}, nil
36
33
}
37
34
38
-
feeds := discoverFeedLinks(ctx, discoverClient, siteURL)
39
-
result.FeedURLs = feeds
35
+
if m := baseHrefRe.FindStringSubmatch(html); len(m) >= 2 && m[1] != "" {
36
+
if u, err := base.Parse(m[1]); err == nil {
37
+
base = u
38
+
}
39
+
}
40
40
41
-
return result, nil
41
+
links := linkRe.FindAllString(html, -1)
42
+
return &DiscoveryResult{
43
+
FeedURLs: findFeedURLs(base, links),
44
+
Favicon: findFavicon(ctx, base, links),
45
+
}, nil
42
46
}
43
47
44
-
func discoverFeedLinks(ctx context.Context, client *http.Client, siteURL string) []string {
45
-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
46
-
if err != nil {
47
-
return nil
48
-
}
49
-
req.Header.Set("Accept", "text/html")
50
-
51
-
resp, err := client.Do(req)
52
-
if err != nil {
53
-
return nil
54
-
}
55
-
defer resp.Body.Close()
56
-
57
-
if resp.StatusCode != http.StatusOK {
58
-
return nil
59
-
}
60
-
61
-
body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
62
-
if err != nil {
63
-
return nil
64
-
}
65
-
66
-
html := string(body)
67
-
baseURL := resolveBaseURL(siteURL, html)
68
-
48
+
func findFeedURLs(base *url.URL, links []string) []string {
69
49
var feeds []string
70
-
links := linkRe.FindAllString(html, -1)
71
50
for _, link := range links {
72
51
if !relFeedRe.MatchString(link) && !typeFeedRe.MatchString(link) {
73
52
continue
74
53
}
75
-
76
-
hrefMatch := hrefRe.FindStringSubmatch(link)
77
-
if len(hrefMatch) < 2 {
54
+
href := extractHref(link)
55
+
if href == "" {
78
56
continue
79
57
}
80
-
81
-
feedURL := resolveURL(baseURL, hrefMatch[1])
82
-
if feedURL != "" {
83
-
feeds = append(feeds, feedURL)
58
+
if u, err := base.Parse(href); err == nil {
59
+
feeds = append(feeds, u.String())
84
60
}
85
61
}
86
-
87
62
return feeds
88
63
}
89
64
90
-
func discoverFavicon(ctx context.Context, client *http.Client, siteURL string) string {
91
-
req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
92
-
if err != nil {
93
-
return tryDefaultFavicons(ctx, client, siteURL)
94
-
}
95
-
req.Header.Set("Accept", "text/html")
96
-
97
-
resp, err := client.Do(req)
98
-
if err != nil {
99
-
return tryDefaultFavicons(ctx, client, siteURL)
100
-
}
101
-
defer resp.Body.Close()
102
-
103
-
if resp.StatusCode != http.StatusOK {
104
-
return tryDefaultFavicons(ctx, client, siteURL)
105
-
}
106
-
107
-
body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
108
-
if err != nil {
109
-
return tryDefaultFavicons(ctx, client, siteURL)
110
-
}
111
-
112
-
html := string(body)
113
-
baseURL := resolveBaseURL(siteURL, html)
114
-
115
-
links := linkRe.FindAllString(html, -1)
65
+
func findFavicon(ctx context.Context, base *url.URL, links []string) string {
116
66
for _, link := range links {
117
67
if !relIconRe.MatchString(link) {
118
68
continue
119
69
}
120
-
121
-
hrefMatch := hrefRe.FindStringSubmatch(link)
122
-
if len(hrefMatch) < 2 {
70
+
href := extractHref(link)
71
+
if href == "" {
123
72
continue
124
73
}
125
-
126
-
iconURL := resolveURL(baseURL, hrefMatch[1])
127
-
if iconURL != "" {
128
-
return iconURL
74
+
if u, err := base.Parse(href); err == nil {
75
+
return u.String()
129
76
}
130
77
}
131
78
132
-
return tryDefaultFavicons(ctx, client, siteURL)
133
-
}
134
-
135
-
func tryDefaultFavicons(ctx context.Context, client *http.Client, siteURL string) string {
136
-
parsed := siteURL
137
-
if !strings.HasPrefix(parsed, "http") {
138
-
parsed = "https://" + parsed
139
-
}
140
-
141
-
base := parsed
142
-
idx := strings.Index(base[8:], "/")
143
-
if idx >= 0 {
144
-
base = base[:8+idx]
145
-
}
79
+
origin := *base
80
+
origin.Path = ""
81
+
origin.RawQuery = ""
82
+
origin.Fragment = ""
146
83
147
84
for _, path := range faviconPaths {
148
-
url := base + path
149
-
req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil)
85
+
u, _ := url.Parse(path)
86
+
resolved := origin.ResolveReference(u)
87
+
req, err := http.NewRequestWithContext(ctx, http.MethodHead, resolved.String(), nil)
150
88
if err != nil {
151
89
continue
152
90
}
153
-
154
-
resp, err := client.Do(req)
91
+
resp, err := discoverClient.Do(req)
155
92
if err != nil {
156
93
continue
157
94
}
158
95
resp.Body.Close()
159
-
160
96
if resp.StatusCode == http.StatusOK {
161
-
return url
97
+
return resolved.String()
162
98
}
163
99
}
164
-
165
100
return ""
166
101
}
167
102
168
-
func resolveBaseURL(siteURL, html string) string {
169
-
match := baseHrefRe.FindStringSubmatch(html)
170
-
if len(match) >= 2 && match[1] != "" {
171
-
return resolveURL(siteURL, match[1])
103
+
func extractHref(link string) string {
104
+
m := hrefRe.FindStringSubmatch(link)
105
+
if len(m) >= 2 {
106
+
return m[1]
172
107
}
173
-
return siteURL
108
+
return ""
174
109
}
175
110
176
-
func resolveURL(base, ref string) string {
177
-
if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
178
-
return ref
111
+
func fetchHTML(ctx context.Context, siteURL string) (*url.URL, string) {
112
+
req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil)
113
+
if err != nil {
114
+
return nil, ""
179
115
}
116
+
req.Header.Set("Accept", "text/html")
180
117
181
-
base = strings.TrimRight(base, "/")
182
-
if strings.HasPrefix(ref, "//") {
183
-
return "https:" + ref
118
+
resp, err := discoverClient.Do(req)
119
+
if err != nil {
120
+
return nil, ""
184
121
}
185
-
if strings.HasPrefix(ref, "/") {
186
-
idx := strings.Index(base[8:], "/")
187
-
if idx >= 0 {
188
-
return base[:8+idx] + ref
189
-
}
190
-
return base + ref
122
+
defer resp.Body.Close()
123
+
124
+
if resp.StatusCode != http.StatusOK {
125
+
return nil, ""
191
126
}
192
127
193
-
idx := strings.LastIndex(base, "/")
194
-
if idx > 8 {
195
-
return base[:idx+1] + ref
128
+
body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512))
129
+
if err != nil {
130
+
return nil, ""
196
131
}
197
-
return fmt.Sprintf("%s/%s", base, ref)
132
+
133
+
base := resp.Request.URL
134
+
return base, string(body)
198
135
}
+10
-55
internal/server/auth_handler.go
+10
-55
internal/server/auth_handler.go
···
3
3
import (
4
4
"context"
5
5
"encoding/json"
6
-
"fmt"
7
6
"net/http"
8
-
"net/url"
9
7
"strings"
10
-
"os"
11
8
12
9
"github.com/bluesky-social/indigo/atproto/syntax"
13
10
···
95
92
handle = ident.Handle.String()
96
93
}
97
94
98
-
displayName, avatarURL := s.fetchUserProfile(r.Context(), sessData)
95
+
client := s.pdsClientFromSession(sessData)
96
+
97
+
var displayName, avatarURL string
98
+
if client != nil {
99
+
if dn, avatar, err := client.GetProfile(r.Context(), did); err == nil {
100
+
displayName = dn
101
+
avatarURL = avatar
102
+
}
103
+
}
99
104
100
105
user, err := s.db.CreateUser(r.Context(), did, handle, displayName, avatarURL)
101
106
if err != nil {
···
125
130
SameSite: http.SameSiteLaxMode,
126
131
})
127
132
128
-
s.syncUserInBackground(user.DID, s.pdsClientFromSession(sessData))
133
+
s.syncUserInBackground(user.DID, client)
129
134
130
135
http.Redirect(w, r, "/dashboard", http.StatusSeeOther)
131
136
}
132
137
133
-
func (s *Server) fetchUserProfile(ctx context.Context, sessData *oauth.ClientSessionData) (string, string) {
134
-
did := sessData.AccountDID.String()
135
-
136
-
session, err := s.oauth.ResumeSession(ctx, sessData.AccountDID, sessData.SessionID)
137
-
if err != nil {
138
-
s.logger.Warn("failed to resume session for profile fetch", "error", err)
139
-
return "", ""
140
-
}
141
-
142
-
var profile struct {
143
-
DisplayName string `json:"displayName"`
144
-
Avatar string `json:"avatar"`
145
-
}
146
-
147
-
nsid, _ := syntax.ParseNSID("app.bsky.actor.getProfile")
148
-
if err := session.APIClient().Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil {
149
-
s.logger.Warn("failed to fetch profile from PDS", "error", err, "did", did)
150
-
return "", ""
151
-
}
152
-
return profile.DisplayName, profile.Avatar
153
-
}
154
-
155
138
func (s *Server) pdsClientFromSession(sessData *oauth.ClientSessionData) *atproto.Client {
156
139
session, err := s.oauth.ResumeSession(context.Background(), sessData.AccountDID, sessData.SessionID)
157
140
if err != nil {
···
188
171
s.clearUserSession(w)
189
172
http.Redirect(w, r, "/", http.StatusSeeOther)
190
173
}
191
-
192
-
func resolveCallbackURL(r *http.Request) string {
193
-
scheme := "http"
194
-
if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
195
-
scheme = "https"
196
-
}
197
-
return fmt.Sprintf("%s://%s/auth/callback", scheme, r.Host)
198
-
}
199
-
200
-
func resolveClientID(r *http.Request) string {
201
-
cid := os.Getenv("GLEAN_OAUTH_CLIENT_ID")
202
-
if cid != "" {
203
-
return cid
204
-
}
205
-
scheme := "http"
206
-
if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
207
-
scheme = "https"
208
-
}
209
-
return fmt.Sprintf("%s://%s/oauth/client-metadata", scheme, r.Host)
210
-
}
211
-
212
-
func resolveBaseURL(r *http.Request) *url.URL {
213
-
scheme := "http"
214
-
if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" {
215
-
scheme = "https"
216
-
}
217
-
return &url.URL{Scheme: scheme, Host: r.Host}
218
-
}
+12
-1
internal/server/server.go
+12
-1
internal/server/server.go
···
490
490
handle = ident.Handle.String()
491
491
}
492
492
493
-
if _, err := s.db.CreateUser(ctx, did, handle, "", ""); err != nil {
493
+
var displayName, avatarURL string
494
+
if _, dn, avatar, err := atproto.FetchProfile(ctx, did); err == nil {
495
+
displayName = dn
496
+
avatarURL = avatar
497
+
}
498
+
499
+
if _, err := s.db.CreateUser(ctx, did, handle, displayName, avatarURL); err != nil {
494
500
s.logger.Error("failed to create user during backfill", "error", err, "did", did)
495
501
return
496
502
}
···
545
551
metrics.SyncErrors.Inc()
546
552
s.logger.Error("periodic sync failed", "error", err, "did", u.DID)
547
553
}
554
+
555
+
if dn, avatar, err := client.GetProfile(ctx, u.DID); err == nil {
556
+
_ = s.db.UpdateUserProfile(ctx, u.DID, dn, avatar)
557
+
}
558
+
548
559
metrics.SyncRuns.Inc()
549
560
}
550
561
}