tymek.me / slurp
Homebrew RSS reader server
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

slurp: add OPML import

Tim Chmielecki ff287485 c50a2391

+199 -290
+2
.gitignore
··· 2 2 *.db 3 3 *.db-wal 4 4 *.db-shm 5 + slurp.toml 6 + SUMMARY.md
+103 -23
Cargo.lock
··· 270 270 "heck", 271 271 "proc-macro2", 272 272 "quote", 273 - "syn", 273 + "syn 2.0.114", 274 274 ] 275 275 276 276 [[package]] ··· 386 386 dependencies = [ 387 387 "proc-macro2", 388 388 "quote", 389 - "syn", 389 + "syn 2.0.114", 390 390 ] 391 391 392 392 [[package]] ··· 610 610 ] 611 611 612 612 [[package]] 613 + name = "hard-xml" 614 + version = "1.41.0" 615 + source = "registry+https://github.com/rust-lang/crates.io-index" 616 + checksum = "3b07b8ba970e18a03dbb79f6786b6e4d6f198a0ac839aa5182017001bb8dee17" 617 + dependencies = [ 618 + "hard-xml-derive", 619 + "jetscii", 620 + "lazy_static", 621 + "memchr", 622 + "xmlparser", 623 + ] 624 + 625 + [[package]] 626 + name = "hard-xml-derive" 627 + version = "1.41.0" 628 + source = "registry+https://github.com/rust-lang/crates.io-index" 629 + checksum = "f0c43e7c3212bd992c11b6b9796563388170950521ae8487f5cdf6f6e792f1c8" 630 + dependencies = [ 631 + "bitflags", 632 + "proc-macro2", 633 + "quote", 634 + "syn 1.0.109", 635 + ] 636 + 637 + [[package]] 613 638 name = "hashbrown" 614 639 version = "0.15.5" 615 640 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 947 972 checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" 948 973 949 974 [[package]] 975 + name = "jetscii" 976 + version = "0.5.3" 977 + source = "registry+https://github.com/rust-lang/crates.io-index" 978 + checksum = "47f142fe24a9c9944451e8349de0a56af5f3e7226dc46f3ed4d4ecc0b85af75e" 979 + 980 + [[package]] 950 981 name = "js-sys" 951 982 version = "0.3.85" 952 983 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1142 1173 checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" 1143 1174 1144 1175 [[package]] 1176 + name = "opml" 1177 + version = "1.1.6" 1178 + source = "registry+https://github.com/rust-lang/crates.io-index" 1179 + checksum = "df2f96426c857a92676dc29a9e2a181eb39321047ac994491c69eae01619ddf2" 1180 + dependencies = [ 1181 + "hard-xml", 1182 + "serde", 1183 + "thiserror 1.0.69", 1184 + ] 1185 + 1186 + [[package]] 1145 1187 name = "parking" 1146 1188 version = "2.2.1" 1147 1189 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 1275 1317 "rustc-hash", 1276 1318 "rustls", 1277 1319 "socket2", 1278 - "thiserror", 1320 + "thiserror 2.0.18", 1279 1321 "tokio", 1280 1322 "tracing", 1281 1323 "web-time", ··· 1296 1338 "rustls", 1297 1339 "rustls-pki-types", 1298 1340 "slab", 1299 - "thiserror", 1341 + "thiserror 2.0.18", 1300 1342 "tinyvec", 1301 1343 "tracing", 1302 1344 "web-time", ··· 1595 1637 dependencies = [ 1596 1638 "proc-macro2", 1597 1639 "quote", 1598 - "syn", 1640 + "syn 2.0.114", 1599 1641 ] 1600 1642 1601 1643 [[package]] ··· 1723 1765 "clap", 1724 1766 "feed-rs", 1725 1767 "md-5", 1768 + "opml", 1726 1769 "reqwest", 1727 1770 "serde", 1728 1771 "serde_json", ··· 1813 1856 "serde_json", 1814 1857 "sha2", 1815 1858 "smallvec", 1816 - "thiserror", 1859 + "thiserror 2.0.18", 1817 1860 "tokio", 1818 1861 "tokio-stream", 1819 1862 "tracing", ··· 1830 1873 "quote", 1831 1874 "sqlx-core", 1832 1875 "sqlx-macros-core", 1833 - "syn", 1876 + "syn 2.0.114", 1834 1877 ] 1835 1878 1836 1879 [[package]] ··· 1853 1896 "sqlx-mysql", 1854 1897 "sqlx-postgres", 1855 1898 "sqlx-sqlite", 1856 - "syn", 1899 + "syn 2.0.114", 1857 1900 "tokio", 1858 1901 "url", 1859 1902 ] ··· 1895 1938 "smallvec", 1896 1939 "sqlx-core", 1897 1940 "stringprep", 1898 - "thiserror", 1941 + "thiserror 2.0.18", 1899 1942 "tracing", 1900 1943 "whoami", 1901 1944 ] ··· 1932 1975 "smallvec", 1933 1976 "sqlx-core", 1934 1977 "stringprep", 1935 - "thiserror", 1978 + "thiserror 2.0.18", 1936 1979 "tracing", 1937 1980 "whoami", 1938 1981 ] ··· 1956 1999 "serde", 1957 2000 "serde_urlencoded", 1958 2001 "sqlx-core", 1959 - "thiserror", 2002 + "thiserror 2.0.18", 1960 2003 "tracing", 1961 2004 "url", 1962 2005 ] ··· 1992 2035 1993 2036 [[package]] 1994 2037 name = "syn" 2038 + version = "1.0.109" 2039 + source = "registry+https://github.com/rust-lang/crates.io-index" 2040 + checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 2041 + dependencies = [ 2042 + "proc-macro2", 2043 + "quote", 2044 + "unicode-ident", 2045 + ] 2046 + 2047 + [[package]] 2048 + name = "syn" 1995 2049 version = "2.0.114" 1996 2050 source = "registry+https://github.com/rust-lang/crates.io-index" 1997 2051 checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" ··· 2018 2072 dependencies = [ 2019 2073 "proc-macro2", 2020 2074 "quote", 2021 - "syn", 2075 + "syn 2.0.114", 2076 + ] 2077 + 2078 + [[package]] 2079 + name = "thiserror" 2080 + version = "1.0.69" 2081 + source = "registry+https://github.com/rust-lang/crates.io-index" 2082 + checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" 2083 + dependencies = [ 2084 + "thiserror-impl 1.0.69", 2022 2085 ] 2023 2086 2024 2087 [[package]] ··· 2027 2090 source = "registry+https://github.com/rust-lang/crates.io-index" 2028 2091 checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" 2029 2092 dependencies = [ 2030 - "thiserror-impl", 2093 + "thiserror-impl 2.0.18", 2094 + ] 2095 + 2096 + [[package]] 2097 + name = "thiserror-impl" 2098 + version = "1.0.69" 2099 + source = "registry+https://github.com/rust-lang/crates.io-index" 2100 + checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" 2101 + dependencies = [ 2102 + "proc-macro2", 2103 + "quote", 2104 + "syn 2.0.114", 2031 2105 ] 2032 2106 2033 2107 [[package]] ··· 2038 2112 dependencies = [ 2039 2113 "proc-macro2", 2040 2114 "quote", 2041 - "syn", 2115 + "syn 2.0.114", 2042 2116 ] 2043 2117 2044 2118 [[package]] ··· 2100 2174 dependencies = [ 2101 2175 "proc-macro2", 2102 2176 "quote", 2103 - "syn", 2177 + "syn 2.0.114", 2104 2178 ] 2105 2179 2106 2180 [[package]] ··· 2231 2305 dependencies = [ 2232 2306 "proc-macro2", 2233 2307 "quote", 2234 - "syn", 2308 + "syn 2.0.114", 2235 2309 ] 2236 2310 2237 2311 [[package]] ··· 2444 2518 "bumpalo", 2445 2519 "proc-macro2", 2446 2520 "quote", 2447 - "syn", 2521 + "syn 2.0.114", 2448 2522 "wasm-bindgen-shared", 2449 2523 ] 2450 2524 ··· 2517 2591 dependencies = [ 2518 2592 "proc-macro2", 2519 2593 "quote", 2520 - "syn", 2594 + "syn 2.0.114", 2521 2595 ] 2522 2596 2523 2597 [[package]] ··· 2528 2602 dependencies = [ 2529 2603 "proc-macro2", 2530 2604 "quote", 2531 - "syn", 2605 + "syn 2.0.114", 2532 2606 ] 2533 2607 2534 2608 [[package]] ··· 2799 2873 checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" 2800 2874 2801 2875 [[package]] 2876 + name = "xmlparser" 2877 + version = "0.13.6" 2878 + source = "registry+https://github.com/rust-lang/crates.io-index" 2879 + checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" 2880 + 2881 + [[package]] 2802 2882 name = "yoke" 2803 2883 version = "0.8.1" 2804 2884 source = "registry+https://github.com/rust-lang/crates.io-index" ··· 2817 2897 dependencies = [ 2818 2898 "proc-macro2", 2819 2899 "quote", 2820 - "syn", 2900 + "syn 2.0.114", 2821 2901 "synstructure", 2822 2902 ] 2823 2903 ··· 2838 2918 dependencies = [ 2839 2919 "proc-macro2", 2840 2920 "quote", 2841 - "syn", 2921 + "syn 2.0.114", 2842 2922 ] 2843 2923 2844 2924 [[package]] ··· 2858 2938 dependencies = [ 2859 2939 "proc-macro2", 2860 2940 "quote", 2861 - "syn", 2941 + "syn 2.0.114", 2862 2942 "synstructure", 2863 2943 ] 2864 2944 ··· 2898 2978 dependencies = [ 2899 2979 "proc-macro2", 2900 2980 "quote", 2901 - "syn", 2981 + "syn 2.0.114", 2902 2982 ] 2903 2983 2904 2984 [[package]]
+1
Cargo.toml
··· 19 19 tracing-subscriber = "0.3" 20 20 md-5 = "0.10" 21 21 base64 = "0.22" 22 + opml = "1.1" 22 23 chrono = "0.4" 23 24 anyhow = "1"
-251
SUMMARY.md
··· 1 - ## Project Summary: Custom RSS Reader 2 - 3 - ### Goals 4 - 5 - Build a lightweight, self-hosted RSS aggregator in Rust that serves feeds via the Fever API to external RSS readers (ReadKit on iOS). No web interface needed - the server is headless and solely provides API endpoints for feed consumption. 6 - 7 - ### Core Requirements 8 - 9 - **Feed Aggregation:** 10 - 11 - - Add RSS/Atom feeds by URL 12 - - Automatic polling and fetching (15min intervals) 13 - - Deduplication and storage in SQLite 14 - - Serve items chronologically 15 - 16 - **Email-to-RSS:** 17 - 18 - - Generate unique email addresses per newsletter "feed" 19 - - Accept incoming emails via SMTP or webhook 20 - - Parse newsletter content and extract links 21 - - Store as RSS items alongside regular feeds 22 - 23 - **API Compatibility:** 24 - 25 - - Implement Fever API (ReadKit supports this) 26 - - JSON responses with groups, feeds, items 27 - - Authentication via API key 28 - - No read/unread state tracking (client handles this) 29 - 30 - **Deployment:** 31 - 32 - - NixOS service on bigchungus (Minisforum UM790) 33 - - SQLite database in `/data/rss` 34 - - Exposed via Caddy reverse proxy 35 - - No web UI - purely API-driven 36 - 37 - ### Non-Requirements 38 - 39 - - Web interface for reading feeds 40 - - User accounts or multi-user support (single user) 41 - - Read/unread state synchronization 42 - - Mobile push notifications (ReadKit handles this) 43 - - Feed discovery or search features 44 - - Complex filtering or tagging 45 - 46 - ### Success Criteria 47 - 48 - ReadKit can connect, fetch all feeds (RSS + newsletters), and display items - all without touching the server's terminal. Client maintains its own read state locally. 49 - 50 - ## Architecture overview 51 - 52 - Looking at ReadKit's supported services, **Fever API** is the simplest to implement - it's a straightforward JSON API that many RSS readers support. 53 - 54 - ## Implementation Plan 55 - 56 - ### Architecture 57 - 58 - ``` 59 - ┌─────────────┐ 60 - │ ReadKit │──────HTTP───────┐ 61 - │ (iOS) │ │ 62 - └─────────────┘ ▼ 63 - ┌──────────────┐ 64 - │ Rust Server │ 65 - │ (axum) │ 66 - └──────┬───────┘ 67 - │ 68 - ┌───────────┼───────────┐ 69 - ▼ ▼ ▼ 70 - ┌─────────┐ ┌─────────┐ ┌─────────┐ 71 - │ SQLite │ │ Fetcher│ │ Email │ 72 - │ DB │ │ (cron) │ │ Handler │ 73 - └─────────┘ └─────────┘ └─────────┘ 74 - ``` 75 - 76 - ### Tech Stack 77 - 78 - | Component | Choice | Why | 79 - | ------------- | ------------------------ | ---------------------------------------- | 80 - | Web framework | **axum** | Ergonomic, Tower ecosystem, tokio-native | 81 - | DB | **SQLite** | Simple, NixOS-friendly, plenty for RSS | 82 - | RSS parsing | **feed-rs** | Best maintained Rust RSS/Atom parser | 83 - | HTTP client | **reqwest** | Standard choice | 84 - | Email parsing | **mail-parser** | Clean MIME parsing | 85 - | Scheduler | **tokio-cron-scheduler** | Background feed fetching | 86 - 87 - ### Phase 1: Core RSS Aggregator (1-2 days) 88 - 89 - **DB Schema:** 90 - 91 - ```sql 92 - CREATE TABLE feeds ( 93 - id INTEGER PRIMARY KEY, 94 - url TEXT UNIQUE NOT NULL, 95 - title TEXT, 96 - last_fetched INTEGER 97 - ); 98 - 99 - CREATE TABLE items ( 100 - id INTEGER PRIMARY KEY, 101 - feed_id INTEGER, 102 - guid TEXT UNIQUE NOT NULL, 103 - title TEXT, 104 - url TEXT, 105 - content TEXT, 106 - published INTEGER, 107 - is_read BOOLEAN DEFAULT 0 108 - ); 109 - ``` 110 - 111 - **Fever API endpoints to implement:** 112 - 113 - - `GET /fever/?api` - Returns groups, feeds, items 114 - - `POST /fever/?api&mark=item&as=read&id=X` - Mark as read 115 - - Auth via `?api_key=<hash>` 116 - 117 - **Tasks:** 118 - 119 - 1. Set up axum server with Fever routes 120 - 2. RSS fetcher that polls feeds every 15min 121 - 3. Basic dedup (by GUID) 122 - 4. Serve items via Fever JSON format 123 - 124 - ### Phase 2: Email-to-RSS (1 day) 125 - 126 - **Approach:** Generate unique email addresses per "feed" 127 - 128 - ``` 129 - newsletter-<uuid>@yourdomain.com 130 - ``` 131 - 132 - **Flow:** 133 - 134 - 1. User creates a newsletter "feed" → get email address 135 - 2. Forward newsletters to that address (via email forwarding rules or MX records) 136 - 3. Server receives email via SMTP → parse → store as RSS item 137 - 4. ReadKit sees it as another feed 138 - 139 - **Email handler:** 140 - 141 - ```rust 142 - // Parse incoming email 143 - let parsed = mail_parser::Message::parse(&raw_email)?; 144 - 145 - // Extract links from HTML body 146 - let html = parsed.html_body()?; 147 - let links = extract_links(html); // scraper crate 148 - 149 - // Create RSS item 150 - let item = Item { 151 - title: parsed.subject(), 152 - content: format_links_as_html(links), 153 - url: links[0], // first link as canonical 154 - published: parsed.date(), 155 - ... 156 - }; 157 - ``` 158 - 159 - ### Phase 3: NixOS Module (half day) 160 - 161 - ```nix 162 - # /etc/nixos/rss-reader.nix 163 - { config, pkgs, ... }: 164 - 165 - { 166 - systemd.services.rss-reader = { 167 - description = "Custom RSS Reader"; 168 - after = [ "network.target" ]; 169 - wantedBy = [ "multi-user.target" ]; 170 - 171 - serviceConfig = { 172 - ExecStart = "${pkgs.rss-reader}/bin/rss-reader"; 173 - User = "rss"; 174 - WorkingDirectory = "/data/rss"; 175 - Restart = "always"; 176 - }; 177 - }; 178 - 179 - users.users.rss = { 180 - isSystemUser = true; 181 - group = "rss"; 182 - home = "/data/rss"; 183 - }; 184 - 185 - networking.firewall.allowedTCPPorts = [ 8080 ]; 186 - } 187 - ``` 188 - 189 - ### Directory Structure 190 - 191 - ``` 192 - rss-reader/ 193 - ├── Cargo.toml 194 - ├── src/ 195 - │ ├── main.rs 196 - │ ├── api/ 197 - │ │ ├── fever.rs # Fever API implementation 198 - │ │ └── mod.rs 199 - │ ├── db/ 200 - │ │ ├── models.rs 201 - │ │ └── mod.rs 202 - │ ├── fetcher/ 203 - │ │ ├── rss.rs # RSS/Atom fetching 204 - │ │ └── email.rs # Email parsing 205 - │ └── lib.rs 206 - ├── migrations/ 207 - │ └── 001_initial.sql 208 - └── default.nix # Nix package definition 209 - ``` 210 - 211 - ### MVP Feature Checklist 212 - 213 - **Feed Management:** 214 - 215 - - [ ] Add RSS/Atom feed by URL 216 - - [ ] Auto-fetch every 15min 217 - - [ ] Dedupe by GUID 218 - 219 - **Reading:** 220 - 221 - - [ ] Fever API `/fever/?api` endpoint 222 - - [ ] Mark as read/unread 223 - - [ ] Basic auth (api_key in config) 224 - 225 - **Newsletter:** 226 - 227 - - [ ] Generate unique email per feed 228 - - [ ] SMTP server (or webhook receiver) 229 - - [ ] Parse HTML → extract links 230 - - [ ] Store as RSS item 231 - 232 - **NixOS:** 233 - 234 - - [ ] systemd service 235 - - [ ] SQLite in `/data/rss` 236 - - [ ] Reverse proxy via Caddy (already have this) 237 - 238 - ### Quick Start Commands 239 - 240 - ```nix 241 - # In your NixOS config 242 - virtualisation.oci-containers.containers.rss-reader = { 243 - image = "ghcr.io/yourusername/rss-reader:latest"; 244 - ports = [ "127.0.0.1:8080:8080" ]; 245 - volumes = [ "/data/rss:/data" ]; 246 - }; 247 - ``` 248 - 249 - Then point ReadKit to `https://rss.tymek.me/fever/` with your API key. 250 - 251 - Want me to start with a basic `Cargo.toml` and skeleton code for phase 1?
-16
slurp.toml
··· 1 - [server] 2 - bind = "127.0.0.1:8080" 3 - api_key = "83137f7a8fd8a7cae994f09bec2115e1" 4 - 5 - [database] 6 - path = "slurp.db" 7 - 8 - [fetcher] 9 - interval_minutes = 30 10 - 11 - [[groups]] 12 - name = "tech" 13 - 14 - [[feeds]] 15 - url = "https://simonwillison.net/atom/everything/" 16 - group = "tech"
+93
src/main.rs
··· 33 33 #[arg(short, long, default_value = "slurp.toml")] 34 34 config: PathBuf, 35 35 }, 36 + /// Import feeds from an OPML file 37 + Import { 38 + opml_path: PathBuf, 39 + #[arg(short, long, default_value = "slurp.toml")] 40 + config: PathBuf, 41 + }, 36 42 } 37 43 38 44 #[tokio::main] ··· 110 116 111 117 std::fs::write(&config, doc.to_string())?; 112 118 println!("Added feed {} to group {}", url, group); 119 + } 120 + Command::Import { opml_path, config } => { 121 + let opml_content = std::fs::read_to_string(&opml_path)?; 122 + let opml_doc = opml::OPML::from_str(&opml_content)?; 123 + 124 + let content = if config.exists() { 125 + std::fs::read_to_string(&config)? 126 + } else { 127 + r#"[server] 128 + bind = "127.0.0.1:8080" 129 + api_key = "CHANGE_ME" 130 + 131 + [database] 132 + path = "slurp.db" 133 + 134 + [fetcher] 135 + interval_minutes = 30 136 + "# 137 + .to_string() 138 + }; 139 + let mut doc: DocumentMut = content.parse()?; 140 + 141 + if !doc.contains_key("groups") { 142 + doc["groups"] = Item::ArrayOfTables(toml_edit::ArrayOfTables::new()); 143 + } 144 + if !doc.contains_key("feeds") { 145 + doc["feeds"] = Item::ArrayOfTables(toml_edit::ArrayOfTables::new()); 146 + } 147 + 148 + let mut feed_count = 0; 149 + let mut groups_added = std::collections::HashSet::new(); 150 + 151 + fn process_outlines( 152 + outlines: &[opml::Outline], 153 + group_name: &str, 154 + doc: &mut DocumentMut, 155 + feed_count: &mut usize, 156 + groups_added: &mut std::collections::HashSet<String>, 157 + ) { 158 + for outline in outlines { 159 + if let Some(ref url) = outline.xml_url { 160 + // this is a feed 161 + let groups = doc["groups"].as_array_of_tables_mut().unwrap(); 162 + let group_exists = groups 163 + .iter() 164 + .any(|t| t.get("name").and_then(|v| v.as_str()) == Some(group_name)); 165 + if !group_exists { 166 + let mut new_group = Table::new(); 167 + new_group["name"] = toml_edit::value(group_name); 168 + groups.push(new_group); 169 + groups_added.insert(group_name.to_string()); 170 + } 171 + 172 + let feeds = doc["feeds"].as_array_of_tables_mut().unwrap(); 173 + let mut new_feed = Table::new(); 174 + new_feed["url"] = toml_edit::value(url); 175 + new_feed["group"] = toml_edit::value(group_name); 176 + feeds.push(new_feed); 177 + *feed_count += 1; 178 + } else if !outline.outlines.is_empty() { 179 + // this is a group containing feeds 180 + let child_group = outline.text.as_str(); 181 + process_outlines( 182 + &outline.outlines, 183 + child_group, 184 + doc, 185 + feed_count, 186 + groups_added, 187 + ); 188 + } 189 + } 190 + } 191 + 192 + process_outlines( 193 + &opml_doc.body.outlines, 194 + "Uncategorized", 195 + &mut doc, 196 + &mut feed_count, 197 + &mut groups_added, 198 + ); 199 + 200 + std::fs::write(&config, doc.to_string())?; 201 + println!( 202 + "Imported {} feeds into {} groups", 203 + feed_count, 204 + groups_added.len() 205 + ); 113 206 } 114 207 } 115 208