A social RSS reader built on the AT Protocol. glean.at
glean atproto atmosphere rss feed social app
14
fork

Configure Feed

Select the types of activity you want to include in your feed.

Improve profile fetching

+272 -185
+16
internal/atproto/client.go
··· 99 99 } 100 100 return records, result.Cursor, nil 101 101 } 102 + 103 + func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) { 104 + nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile") 105 + if err != nil { 106 + return "", "", fmt.Errorf("parsing NSID: %w", err) 107 + } 108 + 109 + var profile struct { 110 + DisplayName string `json:"displayName"` 111 + Avatar string `json:"avatar"` 112 + } 113 + if err := c.api.Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil { 114 + return "", "", err 115 + } 116 + return profile.DisplayName, profile.Avatar, nil 117 + }
+45
internal/atproto/profile.go
··· 1 + package atproto 2 + 3 + import ( 4 + "context" 5 + "encoding/json" 6 + "fmt" 7 + "net/http" 8 + "net/url" 9 + "time" 10 + ) 11 + 12 + var profileClient = &http.Client{Timeout: 10 * time.Second} 13 + 14 + // FetchProfile is an unauthenticated profile fetcher. It relies on bluesky api. 15 + func FetchProfile(ctx context.Context, identifier string) (handle, displayName, avatarURL string, err error) { 16 + apiURL := fmt.Sprintf("https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=%s", url.QueryEscape(identifier)) 17 + return fetchProfileFromURL(ctx, apiURL, identifier) 18 + } 19 + 20 + func fetchProfileFromURL(ctx context.Context, apiURL, identifier string) (handle, displayName, avatarURL string, err error) { 21 + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) 22 + if err != nil { 23 + return "", "", "", err 24 + } 25 + 26 + resp, err := profileClient.Do(req) 27 + if err != nil { 28 + return "", "", "", err 29 + } 30 + defer resp.Body.Close() 31 + 32 + if resp.StatusCode != http.StatusOK { 33 + return "", "", "", fmt.Errorf("fetch profile %s: status %d", identifier, resp.StatusCode) 34 + } 35 + 36 + var profile struct { 37 + Handle string `json:"handle"` 38 + DisplayName string `json:"displayName"` 39 + Avatar string `json:"avatar"` 40 + } 41 + if err := json.NewDecoder(resp.Body).Decode(&profile); err != nil { 42 + return "", "", "", err 43 + } 44 + return profile.Handle, profile.DisplayName, profile.Avatar, nil 45 + }
+66
internal/atproto/profile_test.go
··· 1 + package atproto 2 + 3 + import ( 4 + "context" 5 + "encoding/json" 6 + "net/http" 7 + "net/http/httptest" 8 + "net/url" 9 + "testing" 10 + 11 + "gotest.tools/v3/assert" 12 + ) 13 + 14 + func TestFetchProfile_Success(t *testing.T) { 15 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 16 + assert.Equal(t, r.URL.Query().Get("actor"), "did:plc:test123") 17 + json.NewEncoder(w).Encode(map[string]string{ 18 + "handle": "test.bsky.social", 19 + "displayName": "Test User", 20 + "avatar": "https://cdn.bsky.app/img/avatar/test.png", 21 + }) 22 + })) 23 + defer srv.Close() 24 + 25 + origClient := profileClient 26 + profileClient = srv.Client() 27 + defer func() { profileClient = origClient }() 28 + 29 + apiURL := srv.URL + "/xrpc/app.bsky.actor.getProfile?actor=" + url.QueryEscape("did:plc:test123") 30 + handle, dn, avatar, err := fetchProfileFromURL(context.Background(), apiURL, "did:plc:test123") 31 + assert.NilError(t, err) 32 + assert.Equal(t, handle, "test.bsky.social") 33 + assert.Equal(t, dn, "Test User") 34 + assert.Equal(t, avatar, "https://cdn.bsky.app/img/avatar/test.png") 35 + } 36 + 37 + func TestFetchProfile_EmptyProfile(t *testing.T) { 38 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 39 + json.NewEncoder(w).Encode(map[string]string{}) 40 + })) 41 + defer srv.Close() 42 + 43 + origClient := profileClient 44 + profileClient = srv.Client() 45 + defer func() { profileClient = origClient }() 46 + 47 + handle, dn, avatar, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123") 48 + assert.NilError(t, err) 49 + assert.Equal(t, handle, "") 50 + assert.Equal(t, dn, "") 51 + assert.Equal(t, avatar, "") 52 + } 53 + 54 + func TestFetchProfile_Non200(t *testing.T) { 55 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 56 + w.WriteHeader(http.StatusNotFound) 57 + })) 58 + defer srv.Close() 59 + 60 + origClient := profileClient 61 + profileClient = srv.Client() 62 + defer func() { profileClient = origClient }() 63 + 64 + _, _, _, err := fetchProfileFromURL(context.Background(), srv.URL+"/xrpc/app.bsky.actor.getProfile", "did:plc:test123") 65 + assert.Assert(t, err != nil) 66 + }
+8 -3
internal/atproto/sync.go
··· 266 266 FollowedAt: db.NullTime(t), 267 267 } 268 268 269 - var handle string 270 - if rec.Subject != userDID { 271 - s.db.CreateUser(ctx, rec.Subject, handle, "", "") 269 + // auto onboard followers 270 + var handle, displayName, avatarURL string 271 + if h, dn, avatar, err := FetchProfile(ctx, rec.Subject); err == nil { 272 + handle = h 273 + displayName = dn 274 + avatarURL = avatar 272 275 } 276 + 277 + s.db.CreateUser(ctx, rec.Subject, handle, displayName, avatarURL) 273 278 } 274 279 275 280 if next == "" || len(records) == 0 {
+11
internal/db/user.go
··· 72 72 return dids, rows.Err() 73 73 } 74 74 75 + func (db *DB) UpdateUserProfile(ctx context.Context, did, displayName, avatarURL string) error { 76 + _, err := db.ExecContext(ctx, ` 77 + UPDATE users SET 78 + display_name = COALESCE(NULLIF(?, ''), display_name), 79 + avatar_url = COALESCE(NULLIF(?, ''), avatar_url), 80 + updated_at = CURRENT_TIMESTAMP 81 + WHERE did = ? 82 + `, displayName, avatarURL, did) 83 + return err 84 + } 85 + 75 86 func (db *DB) ListUsers(ctx context.Context) ([]*User, error) { 76 87 rows, err := db.QueryContext(ctx, ` 77 88 SELECT did, handle, display_name, avatar_url, indexed_at, updated_at
+41
internal/db/user_test.go
··· 1 + package db 2 + 3 + import ( 4 + "context" 5 + "testing" 6 + 7 + "gotest.tools/v3/assert" 8 + ) 9 + 10 + func TestUpdateUserProfile_SetsFields(t *testing.T) { 11 + ctx := context.Background() 12 + db := setupTestDB(t) 13 + 14 + _, err := db.ExecContext(ctx, `INSERT INTO users (did, handle) VALUES (?, ?)`, "did:test:profile", "tester") 15 + assert.NilError(t, err) 16 + 17 + err = db.UpdateUserProfile(ctx, "did:test:profile", "Display Name", "https://cdn.bsky.app/img/avatar.png") 18 + assert.NilError(t, err) 19 + 20 + u, err := db.GetUser(ctx, "did:test:profile") 21 + assert.NilError(t, err) 22 + assert.Equal(t, u.DisplayName.String, "Display Name") 23 + assert.Equal(t, u.AvatarURL.String, "https://cdn.bsky.app/img/avatar.png") 24 + } 25 + 26 + func TestUpdateUserProfile_DoesNotOverwriteWithEmpty(t *testing.T) { 27 + ctx := context.Background() 28 + db := setupTestDB(t) 29 + 30 + _, err := db.ExecContext(ctx, `INSERT INTO users (did, handle, display_name, avatar_url) VALUES (?, ?, ?, ?)`, 31 + "did:test:profile2", "tester2", "Existing Name", "https://old.avatar/url") 32 + assert.NilError(t, err) 33 + 34 + err = db.UpdateUserProfile(ctx, "did:test:profile2", "", "") 35 + assert.NilError(t, err) 36 + 37 + u, err := db.GetUser(ctx, "did:test:profile2") 38 + assert.NilError(t, err) 39 + assert.Equal(t, u.DisplayName.String, "Existing Name") 40 + assert.Equal(t, u.AvatarURL.String, "https://old.avatar/url") 41 + }
+63 -126
internal/feed/discover.go
··· 2 2 3 3 import ( 4 4 "context" 5 - "fmt" 6 5 "io" 7 6 "net/http" 7 + "net/url" 8 8 "regexp" 9 - "strings" 10 9 "time" 11 10 ) 12 11 ··· 16 15 } 17 16 18 17 var ( 19 - linkRe = regexp.MustCompile(`<link[^>]+>`) 20 - hrefRe = regexp.MustCompile(`href="([^"]*)"`) 21 - relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`) 22 - typeFeedRe = regexp.MustCompile(`type="([^"]*(?:rss|atom|feed|xml)[^"]*)"`) 23 - relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`) 24 - baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`) 25 - faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"} 18 + linkRe = regexp.MustCompile(`<link[^>]+>`) 19 + hrefRe = regexp.MustCompile(`href="([^"]*)"`) 20 + relFeedRe = regexp.MustCompile(`rel="(alternate|feed)"`) 21 + typeFeedRe = regexp.MustCompile(`type="[^"]*(?:rss|atom|feed|xml)[^"]*"`) 22 + relIconRe = regexp.MustCompile(`rel="[^"]*icon[^"]*"`) 23 + baseHrefRe = regexp.MustCompile(`<base[^>]+href="([^"]*)"`) 26 24 25 + faviconPaths = []string{"/favicon.ico", "/favicon.png", "/apple-touch-icon.png"} 27 26 discoverClient = &http.Client{Timeout: 15 * time.Second} 28 27 ) 29 28 30 29 func Discover(ctx context.Context, siteURL string) (*DiscoveryResult, error) { 31 - result := &DiscoveryResult{} 32 - 33 - favicon := discoverFavicon(ctx, discoverClient, siteURL) 34 - if favicon != "" { 35 - result.Favicon = favicon 30 + base, html := fetchHTML(ctx, siteURL) 31 + if base == nil { 32 + return &DiscoveryResult{}, nil 36 33 } 37 34 38 - feeds := discoverFeedLinks(ctx, discoverClient, siteURL) 39 - result.FeedURLs = feeds 35 + if m := baseHrefRe.FindStringSubmatch(html); len(m) >= 2 && m[1] != "" { 36 + if u, err := base.Parse(m[1]); err == nil { 37 + base = u 38 + } 39 + } 40 40 41 - return result, nil 41 + links := linkRe.FindAllString(html, -1) 42 + return &DiscoveryResult{ 43 + FeedURLs: findFeedURLs(base, links), 44 + Favicon: findFavicon(ctx, base, links), 45 + }, nil 42 46 } 43 47 44 - func discoverFeedLinks(ctx context.Context, client *http.Client, siteURL string) []string { 45 - req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil) 46 - if err != nil { 47 - return nil 48 - } 49 - req.Header.Set("Accept", "text/html") 50 - 51 - resp, err := client.Do(req) 52 - if err != nil { 53 - return nil 54 - } 55 - defer resp.Body.Close() 56 - 57 - if resp.StatusCode != http.StatusOK { 58 - return nil 59 - } 60 - 61 - body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512)) 62 - if err != nil { 63 - return nil 64 - } 65 - 66 - html := string(body) 67 - baseURL := resolveBaseURL(siteURL, html) 68 - 48 + func findFeedURLs(base *url.URL, links []string) []string { 69 49 var feeds []string 70 - links := linkRe.FindAllString(html, -1) 71 50 for _, link := range links { 72 51 if !relFeedRe.MatchString(link) && !typeFeedRe.MatchString(link) { 73 52 continue 74 53 } 75 - 76 - hrefMatch := hrefRe.FindStringSubmatch(link) 77 - if len(hrefMatch) < 2 { 54 + href := extractHref(link) 55 + if href == "" { 78 56 continue 79 57 } 80 - 81 - feedURL := resolveURL(baseURL, hrefMatch[1]) 82 - if feedURL != "" { 83 - feeds = append(feeds, feedURL) 58 + if u, err := base.Parse(href); err == nil { 59 + feeds = append(feeds, u.String()) 84 60 } 85 61 } 86 - 87 62 return feeds 88 63 } 89 64 90 - func discoverFavicon(ctx context.Context, client *http.Client, siteURL string) string { 91 - req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil) 92 - if err != nil { 93 - return tryDefaultFavicons(ctx, client, siteURL) 94 - } 95 - req.Header.Set("Accept", "text/html") 96 - 97 - resp, err := client.Do(req) 98 - if err != nil { 99 - return tryDefaultFavicons(ctx, client, siteURL) 100 - } 101 - defer resp.Body.Close() 102 - 103 - if resp.StatusCode != http.StatusOK { 104 - return tryDefaultFavicons(ctx, client, siteURL) 105 - } 106 - 107 - body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512)) 108 - if err != nil { 109 - return tryDefaultFavicons(ctx, client, siteURL) 110 - } 111 - 112 - html := string(body) 113 - baseURL := resolveBaseURL(siteURL, html) 114 - 115 - links := linkRe.FindAllString(html, -1) 65 + func findFavicon(ctx context.Context, base *url.URL, links []string) string { 116 66 for _, link := range links { 117 67 if !relIconRe.MatchString(link) { 118 68 continue 119 69 } 120 - 121 - hrefMatch := hrefRe.FindStringSubmatch(link) 122 - if len(hrefMatch) < 2 { 70 + href := extractHref(link) 71 + if href == "" { 123 72 continue 124 73 } 125 - 126 - iconURL := resolveURL(baseURL, hrefMatch[1]) 127 - if iconURL != "" { 128 - return iconURL 74 + if u, err := base.Parse(href); err == nil { 75 + return u.String() 129 76 } 130 77 } 131 78 132 - return tryDefaultFavicons(ctx, client, siteURL) 133 - } 134 - 135 - func tryDefaultFavicons(ctx context.Context, client *http.Client, siteURL string) string { 136 - parsed := siteURL 137 - if !strings.HasPrefix(parsed, "http") { 138 - parsed = "https://" + parsed 139 - } 140 - 141 - base := parsed 142 - idx := strings.Index(base[8:], "/") 143 - if idx >= 0 { 144 - base = base[:8+idx] 145 - } 79 + origin := *base 80 + origin.Path = "" 81 + origin.RawQuery = "" 82 + origin.Fragment = "" 146 83 147 84 for _, path := range faviconPaths { 148 - url := base + path 149 - req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) 85 + u, _ := url.Parse(path) 86 + resolved := origin.ResolveReference(u) 87 + req, err := http.NewRequestWithContext(ctx, http.MethodHead, resolved.String(), nil) 150 88 if err != nil { 151 89 continue 152 90 } 153 - 154 - resp, err := client.Do(req) 91 + resp, err := discoverClient.Do(req) 155 92 if err != nil { 156 93 continue 157 94 } 158 95 resp.Body.Close() 159 - 160 96 if resp.StatusCode == http.StatusOK { 161 - return url 97 + return resolved.String() 162 98 } 163 99 } 164 - 165 100 return "" 166 101 } 167 102 168 - func resolveBaseURL(siteURL, html string) string { 169 - match := baseHrefRe.FindStringSubmatch(html) 170 - if len(match) >= 2 && match[1] != "" { 171 - return resolveURL(siteURL, match[1]) 103 + func extractHref(link string) string { 104 + m := hrefRe.FindStringSubmatch(link) 105 + if len(m) >= 2 { 106 + return m[1] 172 107 } 173 - return siteURL 108 + return "" 174 109 } 175 110 176 - func resolveURL(base, ref string) string { 177 - if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { 178 - return ref 111 + func fetchHTML(ctx context.Context, siteURL string) (*url.URL, string) { 112 + req, err := http.NewRequestWithContext(ctx, http.MethodGet, siteURL, nil) 113 + if err != nil { 114 + return nil, "" 179 115 } 116 + req.Header.Set("Accept", "text/html") 180 117 181 - base = strings.TrimRight(base, "/") 182 - if strings.HasPrefix(ref, "//") { 183 - return "https:" + ref 118 + resp, err := discoverClient.Do(req) 119 + if err != nil { 120 + return nil, "" 184 121 } 185 - if strings.HasPrefix(ref, "/") { 186 - idx := strings.Index(base[8:], "/") 187 - if idx >= 0 { 188 - return base[:8+idx] + ref 189 - } 190 - return base + ref 122 + defer resp.Body.Close() 123 + 124 + if resp.StatusCode != http.StatusOK { 125 + return nil, "" 191 126 } 192 127 193 - idx := strings.LastIndex(base, "/") 194 - if idx > 8 { 195 - return base[:idx+1] + ref 128 + body, err := io.ReadAll(io.LimitReader(resp.Body, 1024*512)) 129 + if err != nil { 130 + return nil, "" 196 131 } 197 - return fmt.Sprintf("%s/%s", base, ref) 132 + 133 + base := resp.Request.URL 134 + return base, string(body) 198 135 }
+10 -55
internal/server/auth_handler.go
··· 3 3 import ( 4 4 "context" 5 5 "encoding/json" 6 - "fmt" 7 6 "net/http" 8 - "net/url" 9 7 "strings" 10 - "os" 11 8 12 9 "github.com/bluesky-social/indigo/atproto/syntax" 13 10 ··· 95 92 handle = ident.Handle.String() 96 93 } 97 94 98 - displayName, avatarURL := s.fetchUserProfile(r.Context(), sessData) 95 + client := s.pdsClientFromSession(sessData) 96 + 97 + var displayName, avatarURL string 98 + if client != nil { 99 + if dn, avatar, err := client.GetProfile(r.Context(), did); err == nil { 100 + displayName = dn 101 + avatarURL = avatar 102 + } 103 + } 99 104 100 105 user, err := s.db.CreateUser(r.Context(), did, handle, displayName, avatarURL) 101 106 if err != nil { ··· 125 130 SameSite: http.SameSiteLaxMode, 126 131 }) 127 132 128 - s.syncUserInBackground(user.DID, s.pdsClientFromSession(sessData)) 133 + s.syncUserInBackground(user.DID, client) 129 134 130 135 http.Redirect(w, r, "/dashboard", http.StatusSeeOther) 131 136 } 132 137 133 - func (s *Server) fetchUserProfile(ctx context.Context, sessData *oauth.ClientSessionData) (string, string) { 134 - did := sessData.AccountDID.String() 135 - 136 - session, err := s.oauth.ResumeSession(ctx, sessData.AccountDID, sessData.SessionID) 137 - if err != nil { 138 - s.logger.Warn("failed to resume session for profile fetch", "error", err) 139 - return "", "" 140 - } 141 - 142 - var profile struct { 143 - DisplayName string `json:"displayName"` 144 - Avatar string `json:"avatar"` 145 - } 146 - 147 - nsid, _ := syntax.ParseNSID("app.bsky.actor.getProfile") 148 - if err := session.APIClient().Get(ctx, nsid, map[string]any{"actor": did}, &profile); err != nil { 149 - s.logger.Warn("failed to fetch profile from PDS", "error", err, "did", did) 150 - return "", "" 151 - } 152 - return profile.DisplayName, profile.Avatar 153 - } 154 - 155 138 func (s *Server) pdsClientFromSession(sessData *oauth.ClientSessionData) *atproto.Client { 156 139 session, err := s.oauth.ResumeSession(context.Background(), sessData.AccountDID, sessData.SessionID) 157 140 if err != nil { ··· 188 171 s.clearUserSession(w) 189 172 http.Redirect(w, r, "/", http.StatusSeeOther) 190 173 } 191 - 192 - func resolveCallbackURL(r *http.Request) string { 193 - scheme := "http" 194 - if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { 195 - scheme = "https" 196 - } 197 - return fmt.Sprintf("%s://%s/auth/callback", scheme, r.Host) 198 - } 199 - 200 - func resolveClientID(r *http.Request) string { 201 - cid := os.Getenv("GLEAN_OAUTH_CLIENT_ID") 202 - if cid != "" { 203 - return cid 204 - } 205 - scheme := "http" 206 - if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { 207 - scheme = "https" 208 - } 209 - return fmt.Sprintf("%s://%s/oauth/client-metadata", scheme, r.Host) 210 - } 211 - 212 - func resolveBaseURL(r *http.Request) *url.URL { 213 - scheme := "http" 214 - if r.TLS != nil || r.Header.Get("X-Forwarded-Proto") == "https" { 215 - scheme = "https" 216 - } 217 - return &url.URL{Scheme: scheme, Host: r.Host} 218 - }
+12 -1
internal/server/server.go
··· 490 490 handle = ident.Handle.String() 491 491 } 492 492 493 - if _, err := s.db.CreateUser(ctx, did, handle, "", ""); err != nil { 493 + var displayName, avatarURL string 494 + if _, dn, avatar, err := atproto.FetchProfile(ctx, did); err == nil { 495 + displayName = dn 496 + avatarURL = avatar 497 + } 498 + 499 + if _, err := s.db.CreateUser(ctx, did, handle, displayName, avatarURL); err != nil { 494 500 s.logger.Error("failed to create user during backfill", "error", err, "did", did) 495 501 return 496 502 } ··· 545 551 metrics.SyncErrors.Inc() 546 552 s.logger.Error("periodic sync failed", "error", err, "did", u.DID) 547 553 } 554 + 555 + if dn, avatar, err := client.GetProfile(ctx, u.DID); err == nil { 556 + _ = s.db.UpdateUserProfile(ctx, u.DID, dn, avatar) 557 + } 558 + 548 559 metrics.SyncRuns.Inc() 549 560 } 550 561 }