Subscribe and post RSS feeds to Bluesky
rss
bluesky
fix: handle first-time feeds and fetch more items, remove fetch limit
+27
-50
+1
-5
internal/rss/feed.go
+1
-5
internal/rss/feed.go
···
30
30
}
31
31
32
32
// FetchLatestItems fetches the latest items from an RSS feed
33
-
func (c *Checker) FetchLatestItems(ctx context.Context, feedURL string, limit int) ([]*FeedItem, error) {
33
+
func (c *Checker) FetchLatestItems(ctx context.Context, feedURL string) ([]*FeedItem, error) {
34
34
feed, err := c.parser.ParseURLWithContext(feedURL, ctx)
35
35
if err != nil {
36
36
return nil, fmt.Errorf("failed to parse RSS feed: %w", err)
···
42
42
43
43
// Limit the number of items to return
44
44
maxItems := len(feed.Items)
45
-
if limit > 0 && limit < maxItems {
46
-
maxItems = limit
47
-
}
48
-
49
45
items := make([]*FeedItem, 0, maxItems)
50
46
for i := 0; i < maxItems; i++ {
51
47
item := feed.Items[i]
+5
-17
internal/rss/feed_test.go
+5
-17
internal/rss/feed_test.go
···
53
53
54
54
// Test fetching all items
55
55
t.Run("FetchAllItems", func(t *testing.T) {
56
-
items, err := checker.FetchLatestItems(context.Background(), server.URL, 0)
56
+
items, err := checker.FetchLatestItems(context.Background(), server.URL)
57
57
if err != nil {
58
58
t.Fatalf("Failed to fetch items: %v", err)
59
59
}
···
74
74
}
75
75
})
76
76
77
-
// Test limiting items
78
-
t.Run("FetchLimitedItems", func(t *testing.T) {
79
-
items, err := checker.FetchLatestItems(context.Background(), server.URL, 2)
80
-
if err != nil {
81
-
t.Fatalf("Failed to fetch items: %v", err)
82
-
}
83
-
84
-
if len(items) != 2 {
85
-
t.Errorf("Expected 2 items, got %d", len(items))
86
-
}
87
-
})
88
-
89
77
// Test with context timeout
90
78
t.Run("ContextTimeout", func(t *testing.T) {
91
79
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond)
···
93
81
94
82
time.Sleep(2 * time.Millisecond) // Ensure context is expired
95
83
96
-
_, err := checker.FetchLatestItems(ctx, server.URL, 0)
84
+
_, err := checker.FetchLatestItems(ctx, server.URL)
97
85
if err == nil {
98
86
t.Error("Expected error with expired context, got nil")
99
87
}
···
103
91
func TestFetchLatestItems_InvalidURL(t *testing.T) {
104
92
checker := NewChecker()
105
93
106
-
_, err := checker.FetchLatestItems(context.Background(), "not-a-valid-url", 0)
94
+
_, err := checker.FetchLatestItems(context.Background(), "not-a-valid-url")
107
95
if err == nil {
108
96
t.Error("Expected error with invalid URL, got nil")
109
97
}
···
127
115
defer server.Close()
128
116
129
117
checker := NewChecker()
130
-
items, err := checker.FetchLatestItems(context.Background(), server.URL, 0)
118
+
items, err := checker.FetchLatestItems(context.Background(), server.URL)
131
119
132
120
if err != nil {
133
121
t.Fatalf("Expected no error with empty feed, got: %v", err)
···
163
151
defer server.Close()
164
152
165
153
checker := NewChecker()
166
-
items, err := checker.FetchLatestItems(context.Background(), server.URL, 0)
154
+
items, err := checker.FetchLatestItems(context.Background(), server.URL)
167
155
168
156
if err != nil {
169
157
t.Fatalf("Failed to fetch items: %v", err)
+21
-28
main.go
+21
-28
main.go
···
61
61
if err != nil {
62
62
log.Fatalf("Failed to initialize storage: %v", err)
63
63
}
64
-
65
-
// Determine if this is the first run (no items in storage)
66
-
isFirstRun := store.Count() == 0
67
-
if isFirstRun {
68
-
log.Println("First run detected - will mark existing items as seen without posting")
69
-
} else {
70
-
log.Printf("Storage initialized with %d previously posted items", store.Count())
71
-
}
64
+
log.Printf("Storage initialized with %d previously posted items", store.Count())
72
65
73
66
rssChecker := rss.NewChecker()
74
67
···
111
104
defer ticker.Stop()
112
105
113
106
// Check immediately on startup
114
-
if err := checkAndPostFeeds(ctx, rssChecker, bskyClient, store, feeds, *dryRun, isFirstRun); err != nil {
107
+
if err := checkAndPostFeeds(ctx, rssChecker, bskyClient, store, feeds, *dryRun); err != nil {
115
108
log.Printf("Error during initial check: %v", err)
116
109
}
117
110
···
121
114
case <-ctx.Done():
122
115
return
123
116
case <-ticker.C:
124
-
if err := checkAndPostFeeds(ctx, rssChecker, bskyClient, store, feeds, *dryRun, false); err != nil {
117
+
if err := checkAndPostFeeds(ctx, rssChecker, bskyClient, store, feeds, *dryRun); err != nil {
125
118
log.Printf("Error during check: %v", err)
126
119
}
127
120
}
···
147
140
return feeds
148
141
}
149
142
150
-
func checkAndPostFeeds(ctx context.Context, rssChecker *rss.Checker, bskyClient *bluesky.Client, store *storage.Storage, feedURLs []string, dryRun bool, isFirstRun bool) error {
143
+
func checkAndPostFeeds(ctx context.Context, rssChecker *rss.Checker, bskyClient *bluesky.Client, store *storage.Storage, feedURLs []string, dryRun bool) error {
151
144
for _, feedURL := range feedURLs {
152
-
if err := checkAndPost(ctx, rssChecker, bskyClient, store, feedURL, dryRun, isFirstRun); err != nil {
145
+
if err := checkAndPost(ctx, rssChecker, bskyClient, store, feedURL, dryRun); err != nil {
153
146
log.Printf("Error checking feed %s: %v", feedURL, err)
154
147
// Continue with other feeds even if one fails
155
148
}
···
158
151
return nil
159
152
}
160
153
161
-
func checkAndPost(
162
-
ctx context.Context,
163
-
rssChecker *rss.Checker,
164
-
bskyClient *bluesky.Client,
165
-
store *storage.Storage,
166
-
feedURL string,
167
-
dryRun bool,
168
-
isFirstRun bool,
169
-
) error {
154
+
func checkAndPost(ctx context.Context, rssChecker *rss.Checker, bskyClient *bluesky.Client, store *storage.Storage, feedURL string, dryRun bool) error {
170
155
log.Printf("Checking RSS feed: %s", feedURL)
171
156
172
-
limit := 20
173
-
items, err := rssChecker.FetchLatestItems(ctx, feedURL, limit)
157
+
items, err := rssChecker.FetchLatestItems(ctx, feedURL)
174
158
if err != nil {
175
159
return fmt.Errorf("failed to fetch RSS items: %w", err)
176
160
}
177
161
178
162
log.Printf("Found %d items in feed", len(items))
179
163
164
+
// Check if this is the first time seeing this feed (no items from it in storage)
165
+
hasSeenFeedBefore := false
166
+
for _, item := range items {
167
+
if store.IsPosted(item.GUID) {
168
+
hasSeenFeedBefore = true
169
+
break
170
+
}
171
+
}
172
+
180
173
// Process items in reverse order (oldest first)
181
174
newItemCount := 0
182
175
postedCount := 0
···
191
184
192
185
newItemCount++
193
186
194
-
// On first run, just mark items as seen without posting
195
-
if isFirstRun {
187
+
// If this is first time seeing this feed, mark items as seen without posting
188
+
if !hasSeenFeedBefore {
196
189
if err := store.MarkPosted(item.GUID); err != nil {
197
190
log.Printf("Failed to mark item as seen: %v", err)
198
191
}
···
228
221
}
229
222
230
223
// Rate limiting - wait a bit between posts to avoid overwhelming Bluesky
231
-
if postedCount > 0 && !dryRun && !isFirstRun {
224
+
if postedCount > 0 && !dryRun {
232
225
time.Sleep(2 * time.Second)
233
226
}
234
227
}
235
228
236
-
if isFirstRun {
229
+
if !hasSeenFeedBefore {
237
230
if newItemCount > 0 {
238
-
log.Printf("Marked %d items as seen from feed %s", newItemCount, feedURL)
231
+
log.Printf("New feed detected: marked %d items as seen from %s (not posted)", newItemCount, feedURL)
239
232
}
240
233
} else {
241
234
if newItemCount == 0 {