rss email digests over ssh because you're a cool kid
herald.dunkirk.sh
go
rss
rss-reader
ssh
charm
fix: prevent duplicate emails on feed file reupload
Changes SCP/SFTP upload handlers from delete-recreate pattern to
update pattern to preserve database IDs and feed history.
Previously, reuploading a feeds file (even unchanged) would:
- Delete the config and all associated feeds (CASCADE DELETE)
- Delete all seen_items history for those feeds
- Create new config/feeds with new IDs
- Cause scheduler to treat all items as new → duplicate emails
Now:
- Checks if config already exists
- Updates existing config instead of deleting
- Matches feeds by URL and updates/adds/removes as needed
- Preserves feed IDs and seen_items history
- No duplicate emails on reupload
Added store methods:
- UpdateConfigTx/UpdateConfig
- GetConfigTx
- UpdateFeedTx/UpdateFeed
- DeleteFeedTx/DeleteFeed
- GetFeedsByConfigTx
Fixes issue where SCP reupload wiped feed history and sent duplicates.
+250
-22
+69
-12
ssh/scp.go
+69
-12
ssh/scp.go
···
156
156
}
157
157
defer func() { _ = tx.Rollback() }()
158
158
159
-
if err := h.store.DeleteConfigTx(ctx, tx, user.ID, name); err != nil {
160
-
h.logger.Debug("no existing config to delete", "filename", name)
161
-
} else {
162
-
h.logger.Debug("deleted existing config", "filename", name)
163
-
}
159
+
// Try to get existing config
160
+
existingCfg, err := h.store.GetConfigTx(ctx, tx, user.ID, name)
161
+
var cfg *store.Config
162
+
163
+
if err == nil {
164
+
// Config exists - update it
165
+
if err := h.store.UpdateConfigTx(ctx, tx, existingCfg.ID, parsed.Email, parsed.CronExpr, parsed.Digest, parsed.Inline, string(content), nextRun); err != nil {
166
+
return 0, fmt.Errorf("failed to update config: %w", err)
167
+
}
168
+
cfg = existingCfg
169
+
cfg.Email = parsed.Email
170
+
cfg.CronExpr = parsed.CronExpr
171
+
cfg.Digest = parsed.Digest
172
+
cfg.InlineContent = parsed.Inline
173
+
cfg.RawText = string(content)
174
+
175
+
// Sync feeds: match by URL, update/delete/add as needed
176
+
existingFeeds, err := h.store.GetFeedsByConfigTx(ctx, tx, cfg.ID)
177
+
if err != nil {
178
+
return 0, fmt.Errorf("failed to get existing feeds: %w", err)
179
+
}
180
+
181
+
// Build maps for comparison
182
+
existingByURL := make(map[string]*store.Feed)
183
+
for _, f := range existingFeeds {
184
+
existingByURL[f.URL] = f
185
+
}
186
+
187
+
newByURL := make(map[string]struct{ URL, Name string })
188
+
for _, f := range parsed.Feeds {
189
+
newByURL[f.URL] = struct{ URL, Name string }{URL: f.URL, Name: f.Name}
190
+
}
191
+
192
+
// Update existing feeds that are still present
193
+
for _, newFeed := range parsed.Feeds {
194
+
if existingFeed, exists := existingByURL[newFeed.URL]; exists {
195
+
// Feed still exists - update name if changed
196
+
if err := h.store.UpdateFeedTx(ctx, tx, existingFeed.ID, newFeed.Name); err != nil {
197
+
return 0, fmt.Errorf("failed to update feed: %w", err)
198
+
}
199
+
} else {
200
+
// New feed - create it
201
+
if _, err := h.store.CreateFeedTx(ctx, tx, cfg.ID, newFeed.URL, newFeed.Name); err != nil {
202
+
return 0, fmt.Errorf("failed to create feed: %w", err)
203
+
}
204
+
}
205
+
}
164
206
165
-
cfg, err := h.store.CreateConfigTx(ctx, tx, user.ID, name, parsed.Email, parsed.CronExpr, parsed.Digest, parsed.Inline, string(content), nextRun)
166
-
if err != nil {
167
-
return 0, fmt.Errorf("failed to save config: %w", err)
168
-
}
207
+
// Delete feeds that are no longer present
208
+
for _, existingFeed := range existingFeeds {
209
+
if _, stillExists := newByURL[existingFeed.URL]; !stillExists {
210
+
if err := h.store.DeleteFeedTx(ctx, tx, existingFeed.ID); err != nil {
211
+
return 0, fmt.Errorf("failed to delete feed: %w", err)
212
+
}
213
+
}
214
+
}
169
215
170
-
for _, feed := range parsed.Feeds {
171
-
if _, err := h.store.CreateFeedTx(ctx, tx, cfg.ID, feed.URL, feed.Name); err != nil {
172
-
return 0, fmt.Errorf("failed to save feed: %w", err)
216
+
h.logger.Debug("updated existing config", "filename", name)
217
+
} else {
218
+
// Config doesn't exist - create new one
219
+
cfg, err = h.store.CreateConfigTx(ctx, tx, user.ID, name, parsed.Email, parsed.CronExpr, parsed.Digest, parsed.Inline, string(content), nextRun)
220
+
if err != nil {
221
+
return 0, fmt.Errorf("failed to create config: %w", err)
173
222
}
223
+
224
+
for _, feed := range parsed.Feeds {
225
+
if _, err := h.store.CreateFeedTx(ctx, tx, cfg.ID, feed.URL, feed.Name); err != nil {
226
+
return 0, fmt.Errorf("failed to create feed: %w", err)
227
+
}
228
+
}
229
+
230
+
h.logger.Debug("created new config", "filename", name)
174
231
}
175
232
176
233
if err := tx.Commit(); err != nil {
+70
-10
ssh/sftp.go
+70
-10
ssh/sftp.go
···
176
176
}
177
177
178
178
ctx := w.handler.session.Context()
179
-
if err := w.handler.store.DeleteConfig(ctx, w.handler.user.ID, w.filename); err != nil {
180
-
w.handler.logger.Debug("no existing config to delete", "filename", w.filename)
181
-
}
182
179
183
-
cfg, err := w.handler.store.CreateConfig(ctx, w.handler.user.ID, w.filename, parsed.Email, parsed.CronExpr, parsed.Digest, parsed.Inline, content, nextRun)
184
-
if err != nil {
185
-
return fmt.Errorf("failed to save config: %w", err)
186
-
}
180
+
// Try to get existing config
181
+
existingCfg, err := w.handler.store.GetConfig(ctx, w.handler.user.ID, w.filename)
182
+
var cfg *store.Config
187
183
188
-
for _, feed := range parsed.Feeds {
189
-
if _, err := w.handler.store.CreateFeed(ctx, cfg.ID, feed.URL, feed.Name); err != nil {
190
-
return fmt.Errorf("failed to save feed: %w", err)
184
+
if err == nil {
185
+
// Config exists - update it
186
+
if err := w.handler.store.UpdateConfig(ctx, existingCfg.ID, parsed.Email, parsed.CronExpr, parsed.Digest, parsed.Inline, content, nextRun); err != nil {
187
+
return fmt.Errorf("failed to update config: %w", err)
188
+
}
189
+
cfg = existingCfg
190
+
cfg.Email = parsed.Email
191
+
cfg.CronExpr = parsed.CronExpr
192
+
cfg.Digest = parsed.Digest
193
+
cfg.InlineContent = parsed.Inline
194
+
cfg.RawText = content
195
+
196
+
// Sync feeds: match by URL, update/delete/add as needed
197
+
existingFeeds, err := w.handler.store.GetFeedsByConfig(ctx, cfg.ID)
198
+
if err != nil {
199
+
return fmt.Errorf("failed to get existing feeds: %w", err)
200
+
}
201
+
202
+
// Build maps for comparison
203
+
existingByURL := make(map[string]*store.Feed)
204
+
for _, f := range existingFeeds {
205
+
existingByURL[f.URL] = f
206
+
}
207
+
208
+
newByURL := make(map[string]struct{ URL, Name string })
209
+
for _, f := range parsed.Feeds {
210
+
newByURL[f.URL] = struct{ URL, Name string }{URL: f.URL, Name: f.Name}
191
211
}
212
+
213
+
// Update existing feeds that are still present
214
+
for _, newFeed := range parsed.Feeds {
215
+
if existingFeed, exists := existingByURL[newFeed.URL]; exists {
216
+
// Feed still exists - update name if changed
217
+
if err := w.handler.store.UpdateFeed(ctx, existingFeed.ID, newFeed.Name); err != nil {
218
+
return fmt.Errorf("failed to update feed: %w", err)
219
+
}
220
+
} else {
221
+
// New feed - create it
222
+
if _, err := w.handler.store.CreateFeed(ctx, cfg.ID, newFeed.URL, newFeed.Name); err != nil {
223
+
return fmt.Errorf("failed to create feed: %w", err)
224
+
}
225
+
}
226
+
}
227
+
228
+
// Delete feeds that are no longer present
229
+
for _, existingFeed := range existingFeeds {
230
+
if _, stillExists := newByURL[existingFeed.URL]; !stillExists {
231
+
if err := w.handler.store.DeleteFeed(ctx, existingFeed.ID); err != nil {
232
+
return fmt.Errorf("failed to delete feed: %w", err)
233
+
}
234
+
}
235
+
}
236
+
237
+
w.handler.logger.Debug("updated existing config via SFTP", "filename", w.filename)
238
+
} else {
239
+
// Config doesn't exist - create new one
240
+
cfg, err = w.handler.store.CreateConfig(ctx, w.handler.user.ID, w.filename, parsed.Email, parsed.CronExpr, parsed.Digest, parsed.Inline, content, nextRun)
241
+
if err != nil {
242
+
return fmt.Errorf("failed to create config: %w", err)
243
+
}
244
+
245
+
for _, feed := range parsed.Feeds {
246
+
if _, err := w.handler.store.CreateFeed(ctx, cfg.ID, feed.URL, feed.Name); err != nil {
247
+
return fmt.Errorf("failed to create feed: %w", err)
248
+
}
249
+
}
250
+
251
+
w.handler.logger.Debug("created new config via SFTP", "filename", w.filename)
192
252
}
193
253
194
254
w.handler.logger.Info("config uploaded via SFTP", "user_id", w.handler.user.ID, "filename", w.filename, "feeds", len(parsed.Feeds))
+35
store/configs.go
+35
store/configs.go
···
82
82
}, nil
83
83
}
84
84
85
+
func (db *DB) UpdateConfigTx(ctx context.Context, tx *sql.Tx, configID int64, email, cronExpr string, digest, inline bool, rawText string, nextRun time.Time) error {
86
+
_, err := tx.ExecContext(ctx,
87
+
`UPDATE configs SET email = ?, cron_expr = ?, digest = ?, inline_content = ?, raw_text = ?, next_run = ? WHERE id = ?`,
88
+
email, cronExpr, digest, inline, rawText, nextRun, configID,
89
+
)
90
+
if err != nil {
91
+
return fmt.Errorf("update config: %w", err)
92
+
}
93
+
return nil
94
+
}
95
+
85
96
func (db *DB) DeleteConfigTx(ctx context.Context, tx *sql.Tx, userID int64, filename string) error {
86
97
result, err := tx.ExecContext(ctx,
87
98
`DELETE FROM configs WHERE user_id = ? AND filename = ?`,
···
110
121
return &cfg, nil
111
122
}
112
123
124
+
func (db *DB) GetConfigTx(ctx context.Context, tx *sql.Tx, userID int64, filename string) (*Config, error) {
125
+
var cfg Config
126
+
err := tx.QueryRowContext(ctx,
127
+
`SELECT id, user_id, filename, email, cron_expr, digest, inline_content, raw_text, last_run, next_run, created_at, last_active_at
128
+
FROM configs WHERE user_id = ? AND filename = ?`,
129
+
userID, filename,
130
+
).Scan(&cfg.ID, &cfg.UserID, &cfg.Filename, &cfg.Email, &cfg.CronExpr, &cfg.Digest, &cfg.InlineContent, &cfg.RawText, &cfg.LastRun, &cfg.NextRun, &cfg.CreatedAt, &cfg.LastActiveAt)
131
+
if err != nil {
132
+
return nil, err
133
+
}
134
+
return &cfg, nil
135
+
}
136
+
113
137
func (db *DB) GetConfigByID(ctx context.Context, id int64) (*Config, error) {
114
138
var cfg Config
115
139
err := db.QueryRowContext(ctx,
···
143
167
configs = append(configs, &cfg)
144
168
}
145
169
return configs, rows.Err()
170
+
}
171
+
172
+
func (db *DB) UpdateConfig(ctx context.Context, configID int64, email, cronExpr string, digest, inline bool, rawText string, nextRun time.Time) error {
173
+
_, err := db.ExecContext(ctx,
174
+
`UPDATE configs SET email = ?, cron_expr = ?, digest = ?, inline_content = ?, raw_text = ?, next_run = ? WHERE id = ?`,
175
+
email, cronExpr, digest, inline, rawText, nextRun, configID,
176
+
)
177
+
if err != nil {
178
+
return fmt.Errorf("update config: %w", err)
179
+
}
180
+
return nil
146
181
}
147
182
148
183
func (db *DB) DeleteConfig(ctx context.Context, userID int64, filename string) error {
+76
store/feeds.go
+76
store/feeds.go
···
71
71
}, nil
72
72
}
73
73
74
+
func (db *DB) UpdateFeedTx(ctx context.Context, tx *sql.Tx, feedID int64, name string) error {
75
+
var nameVal sql.NullString
76
+
if name != "" {
77
+
nameVal = sql.NullString{String: name, Valid: true}
78
+
}
79
+
80
+
_, err := tx.ExecContext(ctx,
81
+
`UPDATE feeds SET name = ? WHERE id = ?`,
82
+
nameVal, feedID,
83
+
)
84
+
if err != nil {
85
+
return fmt.Errorf("update feed: %w", err)
86
+
}
87
+
return nil
88
+
}
89
+
90
+
func (db *DB) DeleteFeedTx(ctx context.Context, tx *sql.Tx, feedID int64) error {
91
+
_, err := tx.ExecContext(ctx,
92
+
`DELETE FROM feeds WHERE id = ?`,
93
+
feedID,
94
+
)
95
+
if err != nil {
96
+
return fmt.Errorf("delete feed: %w", err)
97
+
}
98
+
return nil
99
+
}
100
+
74
101
func (db *DB) GetFeedsByConfig(ctx context.Context, configID int64) ([]*Feed, error) {
75
102
rows, err := db.QueryContext(ctx,
76
103
`SELECT id, config_id, url, name, last_fetched, etag, last_modified
···
93
120
return feeds, rows.Err()
94
121
}
95
122
123
+
func (db *DB) GetFeedsByConfigTx(ctx context.Context, tx *sql.Tx, configID int64) ([]*Feed, error) {
124
+
rows, err := tx.QueryContext(ctx,
125
+
`SELECT id, config_id, url, name, last_fetched, etag, last_modified
126
+
FROM feeds WHERE config_id = ? ORDER BY id`,
127
+
configID,
128
+
)
129
+
if err != nil {
130
+
return nil, fmt.Errorf("query feeds: %w", err)
131
+
}
132
+
defer func() { _ = rows.Close() }()
133
+
134
+
var feeds []*Feed
135
+
for rows.Next() {
136
+
var f Feed
137
+
if err := rows.Scan(&f.ID, &f.ConfigID, &f.URL, &f.Name, &f.LastFetched, &f.ETag, &f.LastModified); err != nil {
138
+
return nil, fmt.Errorf("scan feed: %w", err)
139
+
}
140
+
feeds = append(feeds, &f)
141
+
}
142
+
return feeds, rows.Err()
143
+
}
144
+
96
145
// GetFeedsByConfigs returns a map of configID to feeds for multiple configs in a single query
97
146
func (db *DB) GetFeedsByConfigs(ctx context.Context, configIDs []int64) (map[int64][]*Feed, error) {
98
147
if len(configIDs) == 0 {
···
131
180
}
132
181
133
182
return feedMap, rows.Err()
183
+
}
184
+
185
+
func (db *DB) UpdateFeed(ctx context.Context, feedID int64, name string) error {
186
+
var nameVal sql.NullString
187
+
if name != "" {
188
+
nameVal = sql.NullString{String: name, Valid: true}
189
+
}
190
+
191
+
_, err := db.ExecContext(ctx,
192
+
`UPDATE feeds SET name = ? WHERE id = ?`,
193
+
nameVal, feedID,
194
+
)
195
+
if err != nil {
196
+
return fmt.Errorf("update feed: %w", err)
197
+
}
198
+
return nil
199
+
}
200
+
201
+
func (db *DB) DeleteFeed(ctx context.Context, feedID int64) error {
202
+
_, err := db.ExecContext(ctx,
203
+
`DELETE FROM feeds WHERE id = ?`,
204
+
feedID,
205
+
)
206
+
if err != nil {
207
+
return fmt.Errorf("delete feed: %w", err)
208
+
}
209
+
return nil
134
210
}
135
211
136
212
func (db *DB) UpdateFeedFetched(ctx context.Context, feedID int64, etag, lastModified string) error {