+21

internal/atproto/client.go

reviewed

··· 100 100 return records, result.Cursor, nil 101 101 } 102 102 103 103 + func (c *Client) GetRecord(ctx context.Context, did, collection, rkey string) (json.RawMessage, error) { 104 104 + nsid, err := syntax.ParseNSID("com.atproto.repo.getRecord") 105 105 + if err != nil { 106 106 + return nil, fmt.Errorf("parsing NSID: %w", err) 107 107 + } 108 108 + 109 109 + params := map[string]any{ 110 110 + "repo": did, 111 111 + "collection": collection, 112 112 + "rkey": rkey, 113 113 + } 114 114 + 115 115 + var result struct { 116 116 + Value json.RawMessage `json:"value"` 117 117 + } 118 118 + if err := c.api.Get(ctx, nsid, params, &result); err != nil { 119 119 + return nil, err 120 120 + } 121 121 + return result.Value, nil 122 122 + } 123 123 + 103 124 func (c *Client) GetProfile(ctx context.Context, did string) (displayName, avatarURL string, err error) { 104 125 nsid, err := syntax.ParseNSID("app.bsky.actor.getProfile") 105 126 if err != nil {

+2

internal/atproto/jetstream.go

reviewed

··· 100 100 CollectionBskyFollow, 101 101 CollectionTangledFollow, 102 102 CollectionMarginNote, 103 103 + CollectionStandardPublication, 104 104 + CollectionStandardDocument, 103 105 }, 104 106 } 105 107

+2

internal/atproto/lexicon.go

reviewed

··· 15 15 CollectionSkyreaderSubscription = "app.skyreader.feed.subscription" 16 16 CollectionBskyFollow = "app.bsky.graph.follow" 17 17 CollectionTangledFollow = "sh.tangled.graph.follow" 18 18 + CollectionStandardPublication = "site.standard.publication" 19 19 + CollectionStandardDocument = "site.standard.document" 18 20 ) 19 21 20 22 type SubscriptionRecord struct {

+27

internal/atproto/lexicon_external.go

reviewed

··· 89 89 tags = r.Tags 90 90 return 91 91 } 92 92 + 93 93 + type StandardPublicationRecord struct { 94 94 + BasicTheme json.RawMessage `json:"basicTheme,omitempty"` 95 95 + Name string `json:"name"` 96 96 + URL string `json:"url"` 97 97 + Description string `json:"description"` 98 98 + Icon json.RawMessage `json:"icon,omitempty"` 99 99 + Labels json.RawMessage `json:"labels,omitempty"` 100 100 + Preferences json.RawMessage `json:"preferences,omitempty"` 101 101 + } 102 102 + 103 103 + type StandardDocumentRecord struct { 104 104 + Title string `json:"title"` 105 105 + Site string `json:"site"` 106 106 + Path string `json:"path"` 107 107 + Description string `json:"description"` 108 108 + TextContent string `json:"textContent"` 109 109 + PublishedAt string `json:"publishedAt"` 110 110 + UpdatedAt string `json:"updatedAt"` 111 111 + BskyPostRef json.RawMessage `json:"bskyPostRef,omitempty"` 112 112 + Content json.RawMessage `json:"content,omitempty"` 113 113 + Contributors json.RawMessage `json:"contributors,omitempty"` 114 114 + CoverImage json.RawMessage `json:"coverImage,omitempty"` 115 115 + Labels json.RawMessage `json:"labels,omitempty"` 116 116 + Links json.RawMessage `json:"links,omitempty"` 117 117 + Tags []string `json:"tags,omitempty"` 118 118 + }

+8

internal/atproto/lexicon_test.go

reviewed

··· 93 93 func TestSkyreaderSubscriptionRecordMatchesLexicon(t *testing.T) { 94 94 assertStructMatchesLexiconPath[SkyreaderSubscriptionRecord](t, lexiconPathFromRoot("app/skyreader/feed/subscription.json")) 95 95 } 96 96 + 97 97 + func TestStandardPublicationRecordMatchesLexicon(t *testing.T) { 98 98 + assertStructMatchesLexiconPath[StandardPublicationRecord](t, lexiconPathFromRoot("site/standard/publication.json")) 99 99 + } 100 100 + 101 101 + func TestStandardDocumentRecordMatchesLexicon(t *testing.T) { 102 102 + assertStructMatchesLexiconPath[StandardDocumentRecord](t, lexiconPathFromRoot("site/standard/document.json")) 103 103 + }

+146

internal/atproto/standard_site_fetcher.go

reviewed

··· 1 1 + package atproto 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "encoding/json" 6 6 + "fmt" 7 7 + "log/slog" 8 8 + "strings" 9 9 + "time" 10 10 + 11 11 + "pkg.rbrt.fr/glean/internal/feed" 12 12 + ) 13 13 + 14 14 + func IsATProtoFeedURL(feedURL string) bool { 15 15 + return strings.HasPrefix(feedURL, "at://") 16 16 + } 17 17 + 18 18 + type StandardSiteFetcher struct { 19 19 + logger *slog.Logger 20 20 + } 21 21 + 22 22 + func NewStandardSiteFetcher(logger *slog.Logger) *StandardSiteFetcher { 23 23 + return &StandardSiteFetcher{logger: logger} 24 24 + } 25 25 + 26 26 + func (f *StandardSiteFetcher) FetchFeed(ctx context.Context, feedURL string) (*feed.ParseResult, error) { 27 27 + parsed, ok := ParseRecordURI(feedURL) 28 28 + if !ok { 29 29 + return nil, fmt.Errorf("invalid AT URI: %s", feedURL) 30 30 + } 31 31 + 32 32 + if parsed.Collection != CollectionStandardPublication { 33 33 + return nil, fmt.Errorf("unsupported AT Protocol collection: %s", parsed.Collection) 34 34 + } 35 35 + 36 36 + pdsURL, err := ResolvePDSEndpoint(ctx, parsed.DID) 37 37 + if err != nil { 38 38 + return nil, fmt.Errorf("resolving PDS for %s: %w", parsed.DID, err) 39 39 + } 40 40 + 41 41 + client := NewUnauthenticatedClient(pdsURL) 42 42 + 43 43 + raw, err := client.GetRecord(ctx, parsed.DID, parsed.Collection, parsed.RKey) 44 44 + if err != nil { 45 45 + return nil, fmt.Errorf("fetching publication record: %w", err) 46 46 + } 47 47 + 48 48 + var pub StandardPublicationRecord 49 49 + if err := json.Unmarshal(raw, &pub); err != nil { 50 50 + return nil, fmt.Errorf("parsing publication record: %w", err) 51 51 + } 52 52 + 53 53 + documents, err := f.fetchDocuments(ctx, client, parsed.DID, feedURL, pub.URL) 54 54 + if err != nil { 55 55 + f.logger.Warn("failed to fetch documents for publication", "error", err, "uri", feedURL) 56 56 + } 57 57 + 58 58 + result := &feed.ParseResult{ 59 59 + Feed: feed.Feed{ 60 60 + URL: feedURL, 61 61 + Title: pub.Name, 62 62 + SiteURL: pub.URL, 63 63 + Description: pub.Description, 64 64 + Type: "atproto", 65 65 + }, 66 66 + Articles: documents, 67 67 + } 68 68 + 69 69 + return result, nil 70 70 + } 71 71 + 72 72 + func (f *StandardSiteFetcher) fetchDocuments(ctx context.Context, client *Client, did, publicationURI, publicationURL string) ([]feed.Article, error) { 73 73 + records, err := listAllRecords(ctx, client, did, CollectionStandardDocument) 74 74 + if err != nil { 75 75 + return nil, fmt.Errorf("listing documents: %w", err) 76 76 + } 77 77 + 78 78 + var articles []feed.Article 79 79 + for _, r := range records { 80 80 + var doc StandardDocumentRecord 81 81 + if err := json.Unmarshal(r.Value, &doc); err != nil { 82 82 + continue 83 83 + } 84 84 + if doc.Title == "" { 85 85 + continue 86 86 + } 87 87 + if doc.Site != publicationURI && doc.Site != publicationURL { 88 88 + continue 89 89 + } 90 90 + 91 91 + published := parseRFC3339(doc.PublishedAt) 92 92 + updated := parseRFC3339(doc.UpdatedAt) 93 93 + 94 94 + articles = append(articles, feed.Article{ 95 95 + FeedURL: publicationURI, 96 96 + GUID: r.URI, 97 97 + Title: doc.Title, 98 98 + URL: publicationURL + doc.Path, 99 99 + Author: firstContributor(doc.Contributors), 100 100 + Content: doc.TextContent, 101 101 + Summary: doc.Description, 102 102 + Published: published, 103 103 + Updated: updated, 104 104 + }) 105 105 + } 106 106 + 107 107 + return articles, nil 108 108 + } 109 109 + 110 110 + func firstContributor(raw json.RawMessage) string { 111 111 + if len(raw) == 0 { 112 112 + return "" 113 113 + } 114 114 + var contributors []struct { 115 115 + DisplayName string `json:"displayName"` 116 116 + } 117 117 + if err := json.Unmarshal(raw, &contributors); err != nil { 118 118 + return "" 119 119 + } 120 120 + if len(contributors) == 0 || contributors[0].DisplayName == "" { 121 121 + return "" 122 122 + } 123 123 + return contributors[0].DisplayName 124 124 + } 125 125 + 126 126 + func listAllRecords(ctx context.Context, client *Client, did, collection string) ([]Record, error) { 127 127 + var all []Record 128 128 + cursor := "" 129 129 + for { 130 130 + records, next, err := client.ListRecords(ctx, did, collection, 100, cursor) 131 131 + if err != nil { 132 132 + return nil, err 133 133 + } 134 134 + all = append(all, records...) 135 135 + if next == "" || len(records) == 0 { 136 136 + break 137 137 + } 138 138 + cursor = next 139 139 + } 140 140 + return all, nil 141 141 + } 142 142 + 143 143 + func parseRFC3339(s string) time.Time { 144 144 + t, _ := time.Parse(time.RFC3339, s) 145 145 + return t 146 146 + }

+55

internal/atproto/stream_handler.go

reviewed

··· 9 9 "time" 10 10 11 11 "pkg.rbrt.fr/glean/internal/db" 12 12 + "pkg.rbrt.fr/glean/internal/feed" 12 13 ) 13 14 14 15 const ( ··· 52 53 return h.handleMarginNote(ctx, event) 53 54 case CollectionBskyFollow, CollectionTangledFollow: 54 55 return h.handleFollow(ctx, event) 56 56 + case CollectionStandardDocument: 57 57 + return h.handleStandardDocument(ctx, event) 55 58 } 56 59 return nil 57 60 } ··· 267 270 } 268 271 return article.FeedURL 269 272 } 273 273 + 274 274 + func (h *StreamDBHandler) handleStandardDocument(ctx context.Context, event *Event) error { 275 275 + switch event.Type { 276 276 + case actionCreate, actionUpdate: 277 277 + var doc StandardDocumentRecord 278 278 + if err := json.Unmarshal(event.Value, &doc); err != nil { 279 279 + return err 280 280 + } 281 281 + if doc.Title == "" || doc.Site == "" { 282 282 + return nil 283 283 + } 284 284 + 285 285 + publicationURI := doc.Site 286 286 + if !IsATProtoFeedURL(publicationURI) { 287 287 + return nil 288 288 + } 289 289 + 290 290 + parsed, ok := ParseRecordURI(publicationURI) 291 291 + if !ok || parsed.Collection != CollectionStandardPublication { 292 292 + return nil 293 293 + } 294 294 + 295 295 + published := parseRFC3339(doc.PublishedAt) 296 296 + updated := parseRFC3339(doc.UpdatedAt) 297 297 + 298 298 + _ = h.articles.UpsertFeed(ctx, &db.Feed{ 299 299 + FeedURL: publicationURI, 300 300 + FeedType: sql.NullString{String: "atproto", Valid: true}, 301 301 + }) 302 302 + 303 303 + var articleURL string 304 304 + if f, err := h.articles.GetFeed(ctx, publicationURI); err == nil && f.SiteURL.Valid { 305 305 + articleURL = f.SiteURL.String + doc.Path 306 306 + } 307 307 + 308 308 + articles := []feed.Article{{ 309 309 + FeedURL: publicationURI, 310 310 + GUID: event.URI, 311 311 + Title: doc.Title, 312 312 + URL: articleURL, 313 313 + Content: doc.TextContent, 314 314 + Summary: doc.Description, 315 315 + Published: published, 316 316 + Updated: updated, 317 317 + }} 318 318 + return h.articles.BatchUpsertArticles(ctx, articles) 319 319 + 320 320 + case actionDelete: 321 321 + return h.articles.DeleteArticleByGUID(ctx, event.URI) 322 322 + } 323 323 + return nil 324 324 + }

+5

internal/db/article.go

reviewed

··· 398 398 return a, nil 399 399 } 400 400 401 401 + func (s *ArticleStore) DeleteArticleByGUID(ctx context.Context, guid string) error { 402 402 + _, err := s.db.ExecContext(ctx, `DELETE FROM articles.articles WHERE guid = ?`, guid) 403 403 + return err 404 404 + } 405 405 + 401 406 func (s *ArticleStore) CountNewArticles(ctx context.Context, userDID string, since time.Time) (int, error) { 402 407 var count int 403 408 err := s.db.QueryRowContext(ctx, `

+1 -1

internal/db/db.go

reviewed

··· 262 262 title TEXT, 263 263 site_url TEXT, 264 264 description TEXT, 265 265 - feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json')), 265 265 + feed_type TEXT CHECK(feed_type IN ('rss', 'atom', 'json', 'atproto')), 266 266 last_fetched_at DATETIME, 267 267 last_error TEXT, 268 268 subscriber_count INTEGER NOT NULL DEFAULT 0,

+15 -4

internal/feed/fetcher.go

reviewed

··· 5 5 "fmt" 6 6 "log/slog" 7 7 "net/http" 8 8 + "strings" 8 9 "sync" 9 10 "time" 10 11 ··· 18 19 baseRetryDelay = 1 * time.Second 19 20 ) 20 21 22 22 + type ATProtoFetcher interface { 23 23 + FetchFeed(ctx context.Context, feedURL string) (*ParseResult, error) 24 24 + } 25 25 + 21 26 type Fetcher struct { 22 22 - httpClient *http.Client 27 27 + httpClient *http.Client 28 28 + atprotoFetcher ATProtoFetcher 23 29 } 24 30 25 25 - func NewFetcher() *Fetcher { 31 31 + func NewFetcher(atprotoFetcher ATProtoFetcher) *Fetcher { 26 32 return &Fetcher{ 27 33 httpClient: &http.Client{ 28 34 Timeout: 10 * time.Second, ··· 34 40 return nil 35 41 }, 36 42 }, 43 43 + atprotoFetcher: atprotoFetcher, 37 44 } 38 45 } 39 46 40 47 func (f *Fetcher) Fetch(ctx context.Context, feedURL string) (*ParseResult, error) { 48 48 + if strings.HasPrefix(feedURL, "at://") { 49 49 + return f.atprotoFetcher.FetchFeed(ctx, feedURL) 50 50 + } 51 51 + 41 52 var lastResp *http.Response 42 53 var lastErr error 43 54 ··· 122 133 inFlight sync.Map 123 134 } 124 135 125 125 - func NewScheduler(store FeedStore, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler { 136 136 + func NewScheduler(store FeedStore, atprotoFetcher ATProtoFetcher, logger *slog.Logger, tickInterval, staleInterval time.Duration) *Scheduler { 126 137 return &Scheduler{ 127 127 - fetcher: NewFetcher(), 138 138 + fetcher: NewFetcher(atprotoFetcher), 128 139 store: store, 129 140 logger: logger, 130 141 tickInterval: tickInterval,

+10 -1

internal/server/feeds_handler.go

reviewed

··· 6 6 "errors" 7 7 "fmt" 8 8 "net/http" 9 9 + "net/url" 9 10 "time" 10 11 11 12 "golang.org/x/sync/errgroup" ··· 123 124 return 124 125 } 125 126 127 127 + if !atproto.IsATProtoFeedURL(feedURL) { 128 128 + u, err := url.Parse(feedURL) 129 129 + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { 130 130 + http.Error(w, "invalid feed URL", http.StatusBadRequest) 131 131 + return 132 132 + } 133 133 + } 134 134 + 126 135 result, err := s.fetcher.Fetch(r.Context(), feedURL) 127 127 - if err != nil { 136 136 + if err != nil && !atproto.IsATProtoFeedURL(feedURL) { 128 137 result, feedURL, err = s.discoverFeed(r.Context(), feedURL) 129 138 } 130 139 if err != nil {

+3 -3

internal/server/server.go

reviewed

··· 73 73 sessionKey []byte 74 74 } 75 75 76 76 - func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Scheduler, engine *cluster.Engine, logger *slog.Logger, sessionKey []byte) *Server { 76 76 + func New(dbs *db.Store, clientID, callbackURL, addr string, scheduler *feed.Scheduler, fetcher *feed.Fetcher, engine *cluster.Engine, logger *slog.Logger, sessionKey []byte) *Server { 77 77 oauthStore := db.NewOAuthStore(dbs) 78 78 79 79 var config oauth.ClientConfig ··· 91 91 92 92 s := &Server{ 93 93 dbs: dbs, 94 94 - router: chi.NewRouter(), 94 94 + router: chi.NewMux(), 95 95 logger: logger, 96 96 oauth: oauthClient, 97 97 oauthStore: oauthStore, 98 98 - fetcher: feed.NewFetcher(), 98 98 + fetcher: fetcher, 99 99 scheduler: scheduler, 100 100 engine: engine, 101 101 scraper: scraper.New(logger),

+2 -2

internal/tmpl/feeds.html

reviewed

··· 16 16 function gleanRefreshStart(){document.getElementById('refresh-indicator').style.display='inline';document.getElementById('refresh-btn').disabled=true} 17 17 function gleanRefreshPoll(){setTimeout(function(){htmx.ajax('GET','/feeds/list',{target:'#feed-list',swap:'innerHTML'});document.getElementById('refresh-indicator').style.display='none';document.getElementById('refresh-btn').disabled=false},5000)} 18 18 </script> 19 19 - <p class="text-sm text-spot-secondary mb-6">Manage your RSS and Atom subscriptions.</p> 19 19 + <p class="text-sm text-spot-secondary mb-6">Manage your RSS, Atom, and AT Protocol subscriptions.</p> 20 20 21 21 {{template "dead-feeds.html" (dict "DeadFeeds" .DeadFeeds "CSRFToken" .CSRFToken)}} 22 22 ··· 67 67 hx-on::after-request="if(event.detail.successful)this.reset();else document.getElementById('add-feed-error').textContent=event.detail.xhr.responseText" 68 68 class="space-y-3"> 69 69 {{csrfInput .CSRFToken}} 70 70 - <input type="url" name="feed_url" placeholder="https://example.com/feed.xml" 70 70 + <input type="text" name="feed_url" placeholder="https://example.com/feed.xml or at://did:plc:.../site.standard.publication/..." 71 71 class="w-full bg-spot-hover text-spot-text rounded-pill px-5 py-2 text-sm focus:outline-none focus:ring-2 focus:ring-spot-green placeholder:text-spot-placeholder" 72 72 required> 73 73 <input type="text" name="category" placeholder="Category (optional)"

+125

lexicons/site/standard/document.json

reviewed

··· 1 1 + { 2 2 + "defs": { 3 3 + "contributor": { 4 4 + "properties": { 5 5 + "did": { 6 6 + "format": "did", 7 7 + "type": "string" 8 8 + }, 9 9 + "displayName": { 10 10 + "maxGraphemes": 100, 11 11 + "maxLength": 1000, 12 12 + "type": "string" 13 13 + }, 14 14 + "role": { 15 15 + "maxGraphemes": 100, 16 16 + "maxLength": 1000, 17 17 + "type": "string" 18 18 + } 19 19 + }, 20 20 + "required": [ 21 21 + "did" 22 22 + ], 23 23 + "type": "object" 24 24 + }, 25 25 + "main": { 26 26 + "description": "A document record representing a published article, blog post, or other content. Documents can belong to a publication or exist independently.", 27 27 + "key": "tid", 28 28 + "record": { 29 29 + "properties": { 30 30 + "bskyPostRef": { 31 31 + "description": "Strong reference to a Bluesky post. Useful to keep track of comments off-platform.", 32 32 + "ref": "com.atproto.repo.strongRef", 33 33 + "type": "ref" 34 34 + }, 35 35 + "content": { 36 36 + "closed": false, 37 37 + "description": "Open union used to define the record's content. Each entry must specify a $type and may be extended with other lexicons to support additional content formats.", 38 38 + "refs": [], 39 39 + "type": "union" 40 40 + }, 41 41 + "contributors": { 42 42 + "items": { 43 43 + "ref": "#contributor", 44 44 + "type": "ref" 45 45 + }, 46 46 + "type": "array" 47 47 + }, 48 48 + "coverImage": { 49 49 + "accept": [ 50 50 + "image/*" 51 51 + ], 52 52 + "description": "Image to used for thumbnail or cover image. Less than 1MB is size.", 53 53 + "maxSize": 1000000, 54 54 + "type": "blob" 55 55 + }, 56 56 + "description": { 57 57 + "description": "A brief description or excerpt from the document.", 58 58 + "maxGraphemes": 3000, 59 59 + "maxLength": 30000, 60 60 + "type": "string" 61 61 + }, 62 62 + "labels": { 63 63 + "description": "Self-label values for this post. Effectively content warnings.", 64 64 + "refs": [ 65 65 + "com.atproto.label.defs#selfLabels" 66 66 + ], 67 67 + "type": "union" 68 68 + }, 69 69 + "links": { 70 70 + "description": "Array of values describing relationships between this document and external resources", 71 71 + "refs": [], 72 72 + "type": "union" 73 73 + }, 74 74 + "path": { 75 75 + "description": "Combine with site or publication url to construct a canonical URL to the document. Prepend with a leading slash.", 76 76 + "type": "string" 77 77 + }, 78 78 + "publishedAt": { 79 79 + "description": "Timestamp of the documents publish time.", 80 80 + "format": "datetime", 81 81 + "type": "string" 82 82 + }, 83 83 + "site": { 84 84 + "description": "Points to a publication record (at://) or a publication url (https://) for loose documents. Avoid trailing slashes.", 85 85 + "format": "uri", 86 86 + "type": "string" 87 87 + }, 88 88 + "tags": { 89 89 + "description": "Array of strings used to tag or categorize the document. Avoid prepending tags with hashtags.", 90 90 + "items": { 91 91 + "maxGraphemes": 128, 92 92 + "maxLength": 1280, 93 93 + "type": "string" 94 94 + }, 95 95 + "type": "array" 96 96 + }, 97 97 + "textContent": { 98 98 + "description": "Plaintext representation of the documents contents. Should not contain markdown or other formatting.", 99 99 + "type": "string" 100 100 + }, 101 101 + "title": { 102 102 + "description": "Title of the document.", 103 103 + "maxGraphemes": 500, 104 104 + "maxLength": 5000, 105 105 + "type": "string" 106 106 + }, 107 107 + "updatedAt": { 108 108 + "description": "Timestamp of the documents last edit.", 109 109 + "format": "datetime", 110 110 + "type": "string" 111 111 + } 112 112 + }, 113 113 + "required": [ 114 114 + "site", 115 115 + "title", 116 116 + "publishedAt" 117 117 + ], 118 118 + "type": "object" 119 119 + }, 120 120 + "type": "record" 121 121 + } 122 122 + }, 123 123 + "id": "site.standard.document", 124 124 + "lexicon": 1 125 125 + }

+72

lexicons/site/standard/publication.json

reviewed

··· 1 1 + { 2 2 + "defs": { 3 3 + "main": { 4 4 + "description": "A publication record representing a blog, website, or content platform. Publications serve as containers for documents and define the overall branding and settings.", 5 5 + "key": "tid", 6 6 + "record": { 7 7 + "properties": { 8 8 + "basicTheme": { 9 9 + "description": "Simplified publication theme for tools and apps to utilize when displaying content.", 10 10 + "ref": "site.standard.theme.basic", 11 11 + "type": "ref" 12 12 + }, 13 13 + "description": { 14 14 + "description": "Brief description of the publication.", 15 15 + "maxGraphemes": 3000, 16 16 + "maxLength": 30000, 17 17 + "type": "string" 18 18 + }, 19 19 + "icon": { 20 20 + "accept": [ 21 21 + "image/*" 22 22 + ], 23 23 + "description": "Square image to identify the publication. Should be at least 256x256.", 24 24 + "maxSize": 1000000, 25 25 + "type": "blob" 26 26 + }, 27 27 + "labels": { 28 28 + "description": "Self-label values for this publication. Effectively content warnings.", 29 29 + "refs": [ 30 30 + "com.atproto.label.defs#selfLabels" 31 31 + ], 32 32 + "type": "union" 33 33 + }, 34 34 + "name": { 35 35 + "description": "Name of the publication.", 36 36 + "maxGraphemes": 500, 37 37 + "maxLength": 5000, 38 38 + "type": "string" 39 39 + }, 40 40 + "preferences": { 41 41 + "description": "Object containing platform specific preferences (with a few shared properties).", 42 42 + "ref": "#preferences", 43 43 + "type": "ref" 44 44 + }, 45 45 + "url": { 46 46 + "description": "Base publication url (ex: https://standard.site). The canonical document URL is formed by combining this value with the document path.", 47 47 + "format": "uri", 48 48 + "type": "string" 49 49 + } 50 50 + }, 51 51 + "required": [ 52 52 + "url", 53 53 + "name" 54 54 + ], 55 55 + "type": "object" 56 56 + }, 57 57 + "type": "record" 58 58 + }, 59 59 + "preferences": { 60 60 + "properties": { 61 61 + "showInDiscover": { 62 62 + "default": true, 63 63 + "description": "Boolean which decides whether the publication should appear in discovery feeds.", 64 64 + "type": "boolean" 65 65 + } 66 66 + }, 67 67 + "type": "object" 68 68 + } 69 69 + }, 70 70 + "id": "site.standard.publication", 71 71 + "lexicon": 1 72 72 + }

+4 -2

main.go

reviewed

··· 54 54 callbackURL := envOr("GLEAN_OAUTH_REDIRECT_URL", "") 55 55 56 56 storeAdapter := db.NewFeedAdapter(dbs.Articles) 57 57 - scheduler := feed.NewScheduler(storeAdapter, logger, *fetchInterval, 30*time.Minute) 57 57 + siteFetcher := atproto.NewStandardSiteFetcher(logger) 58 58 + scheduler := feed.NewScheduler(storeAdapter, siteFetcher, logger, *fetchInterval, 30*time.Minute) 58 59 59 60 var embedder cluster.Embedder 60 61 if embedURL := envOr("GLEAN_EMBED_BASE_URL", ""); embedURL != "" { ··· 75 76 76 77 engine := cluster.NewEngine(dbs.SQLDB(), embedder, logger) 77 78 78 78 - srv := server.New(dbs, clientID, callbackURL, *addr, scheduler, engine, logger, []byte(sessionKey)) 79 79 + fetcher := feed.NewFetcher(siteFetcher) 80 80 + srv := server.New(dbs, clientID, callbackURL, *addr, scheduler, fetcher, engine, logger, []byte(sessionKey)) 79 81 80 82 cron := cluster.NewCron(engine, *clusterInterval, logger) 81 83