feat: add photos section, YouTube videos, more blog posts · blaine.bsky.social/ionosphere@7c11598

+18 -1

apps/ionosphere-appview/src/routes.ts

··· 362 362 ORDER BY m.likes DESC` 363 363 ).all(); 364 364 365 + // Photos: posts with images 366 + const photos = db.prepare( 367 + `SELECT m.uri, m.author_did, m.text, m.created_at, m.likes, m.reposts, m.replies, 368 + m.content_type, m.external_url, m.og_title, m.talk_rkey, m.mention_type, 369 + COALESCE(p.handle, m.author_handle) as author_handle, 370 + p.display_name as author_display_name, 371 + p.avatar_url as author_avatar_url, 372 + (SELECT t.title FROM talks t WHERE t.rkey = m.talk_rkey LIMIT 1) as talk_title 373 + FROM mentions m 374 + LEFT JOIN profiles p ON m.author_did = p.did 375 + WHERE m.content_type = 'photo' AND m.parent_uri IS NULL 376 + ORDER BY m.likes DESC` 377 + ).all(); 378 + 365 379 // VOD sites: unique domains from video external_urls 366 380 const vodRows = db.prepare( 367 381 `SELECT DISTINCT m.external_url FROM mentions m ··· 387 401 posts, 388 402 blogs, 389 403 videos, 404 + photos, 390 405 vodSites, 391 406 stats: { 392 407 totalPosts: statsRow?.totalPosts || 0, 393 - blogCount: statsRow?.blogCount || 0, 408 + blogCount: blogs.length, 409 + videoCount: videos.length, 410 + photoCount: photos.length, 394 411 vodSiteCount: vodSites.length, 395 412 uniqueAuthors: statsRow?.uniqueAuthors || 0, 396 413 },

+16 -1

apps/ionosphere/src/app/discussion/DiscussionContent.tsx

··· 54 54 interface Stats { 55 55 totalPosts: number; 56 56 blogCount: number; 57 + photoCount: number; 58 + videoCount: number; 57 59 vodSiteCount: number; 58 60 uniqueAuthors: number; 59 61 } ··· 72 74 | { type: "stats"; stats: Stats } 73 75 | { type: "vodDirectory"; sites: string[] }; 74 76 75 - type FilterKey = "all" | "posts" | "blogs" | "videos"; 77 + type FilterKey = "all" | "posts" | "blogs" | "videos" | "photos"; 76 78 77 79 // --- Height estimation --- 78 80 ··· 155 157 all: [ 156 158 { key: "Top Posts", label: "T" }, 157 159 { key: "Recaps & Blog Posts", label: "R" }, 160 + { key: "Photos", label: "P" }, 158 161 { key: "Videos & VOD Sites", label: "V" }, 159 162 ], 160 163 posts: [{ key: "Top Posts", label: "T" }], 161 164 blogs: [{ key: "Recaps & Blog Posts", label: "R" }], 165 + photos: [{ key: "Photos", label: "P" }], 162 166 videos: [{ key: "Videos & VOD Sites", label: "V" }], 163 167 }; 164 168 ··· 236 240 } 237 241 } 238 242 243 + if (filter === "all" || filter === "photos") { 244 + if ((data as any).photos?.length > 0) { 245 + items.push({ type: "heading", label: "Photos" }); 246 + for (const photo of (data as any).photos) { 247 + items.push({ type: "item", item: photo }); 248 + } 249 + } 250 + } 251 + 239 252 if (filter === "all" || filter === "videos") { 240 253 if (data.videos.length > 0) { 241 254 items.push({ type: "heading", label: "Videos & VOD Sites" }); ··· 387 400 <div className="flex gap-4 flex-wrap"> 388 401 <span>{item.stats.totalPosts.toLocaleString()} posts</span> 389 402 <span>{item.stats.blogCount} recaps</span> 403 + <span>{item.stats.photoCount || 0} photos</span> 390 404 <span>{item.stats.vodSiteCount} VOD sites</span> 391 405 <span>{item.stats.uniqueAuthors} authors</span> 392 406 </div> ··· 480 494 { key: "all" as FilterKey, label: "All" }, 481 495 { key: "posts" as FilterKey, label: "Top Posts" }, 482 496 { key: "blogs" as FilterKey, label: "Recaps & Blog Posts" }, 497 + { key: "photos" as FilterKey, label: "Photos" }, 483 498 { key: "videos" as FilterKey, label: "Videos & VOD Sites" }, 484 499 ]).map((f) => ( 485 500 <button key={f.key} onClick={() => setFilter(f.key)}

+258

scripts/fetch-discussion-extras.mjs

··· 1 + /** 2 + * Fetch additional discussion content: YouTube talks, more blogs, photo posts. 3 + * Run after fetch-discussion.mjs for supplementary content. 4 + */ 5 + 6 + import { createRequire } from 'module'; 7 + const require = createRequire( 8 + new URL('../apps/ionosphere-appview/package.json', import.meta.url).pathname 9 + ); 10 + const { BskyAgent } = require('@atproto/api'); 11 + const Database = require('better-sqlite3'); 12 + 13 + import { fileURLToPath } from 'url'; 14 + import { dirname, join } from 'path'; 15 + 16 + const __dirname = dirname(fileURLToPath(import.meta.url)); 17 + const DB_PATH = join(__dirname, '..', 'apps', 'data', 'ionosphere.sqlite'); 18 + 19 + const agent = new BskyAgent({ service: 'https://bsky.social' }); 20 + function sleep(ms) { return new Promise(r => setTimeout(r, ms)); } 21 + 22 + function extractLinks(post) { 23 + return (post.record?.facets || []) 24 + .flatMap(f => f.features || []) 25 + .filter(f => f.uri) 26 + .map(f => f.uri); 27 + } 28 + 29 + function hasImages(post) { 30 + return !!(post.embed?.images?.length || post.embed?.$type?.includes('image')); 31 + } 32 + 33 + async function fetchOgTitle(url) { 34 + try { 35 + const controller = new AbortController(); 36 + const timeout = setTimeout(() => controller.abort(), 5000); 37 + const res = await fetch(url, { signal: controller.signal, headers: { 'User-Agent': 'ionosphere.tv/1.0' }, redirect: 'follow' }); 38 + clearTimeout(timeout); 39 + if (!res.ok) return null; 40 + const html = await res.text(); 41 + const ogMatch = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i) 42 + || html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:title["']/i); 43 + if (ogMatch) return ogMatch[1]; 44 + const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i); 45 + return titleMatch ? titleMatch[1].trim() : null; 46 + } catch { return null; } 47 + } 48 + 49 + async function main() { 50 + console.log('=== Fetch Discussion Extras ===\n'); 51 + await agent.login({ identifier: 'ionosphere.tv', password: process.env.BOT_PASSWORD }); 52 + 53 + const db = new Database(DB_PATH); 54 + try { db.exec("ALTER TABLE mentions ADD COLUMN has_images INTEGER DEFAULT 0"); } catch {} 55 + 56 + const upsert = db.prepare(` 57 + INSERT INTO mentions (uri, talk_uri, author_did, author_handle, text, created_at, 58 + talk_offset_ms, byte_position, likes, reposts, replies, parent_uri, 59 + mention_type, indexed_at, content_type, external_url, og_title, talk_rkey, has_images) 60 + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) 61 + ON CONFLICT(uri) DO UPDATE SET 62 + likes=excluded.likes, reposts=excluded.reposts, replies=excluded.replies, 63 + content_type=CASE WHEN excluded.content_type != 'post' THEN excluded.content_type ELSE mentions.content_type END, 64 + external_url=COALESCE(excluded.external_url, mentions.external_url), 65 + og_title=COALESCE(excluded.og_title, mentions.og_title), 66 + talk_rkey=COALESCE(excluded.talk_rkey, mentions.talk_rkey), 67 + has_images=CASE WHEN excluded.has_images = 1 THEN 1 ELSE mentions.has_images END, 68 + indexed_at=excluded.indexed_at 69 + `); 70 + 71 + // Load talks for matching 72 + const talks = db.prepare("SELECT DISTINCT rkey, title, uri FROM talks WHERE starts_at IS NOT NULL").all(); 73 + const talksByRkey = new Map(talks.map(t => [t.rkey, t])); 74 + const speakerTalks = db.prepare(` 75 + SELECT s.handle, t.rkey FROM speakers s 76 + JOIN talk_speakers ts ON ts.speaker_uri = s.uri 77 + JOIN talks t ON t.uri = ts.talk_uri WHERE s.handle IS NOT NULL 78 + `).all(); 79 + const speakerToTalks = new Map(); 80 + for (const { handle, rkey } of speakerTalks) { 81 + if (!speakerToTalks.has(handle)) speakerToTalks.set(handle, []); 82 + speakerToTalks.get(handle).push(rkey); 83 + } 84 + 85 + function matchTalk(post, externalUrl) { 86 + if (externalUrl) { 87 + const m = externalUrl.match(/ionosphere\.tv\/talks\/([^/?#]+)/); 88 + if (m && talksByRkey.has(m[1])) return m[1]; 89 + // YouTube — can't match by URL, try speakers 90 + } 91 + const text = post.record?.text || ''; 92 + const handles = text.match(/@([\w.-]+)/g) || []; 93 + for (const h of handles) { 94 + const clean = h.replace('@', ''); 95 + const t = speakerToTalks.get(clean); 96 + if (t?.length === 1) return t[0]; 97 + } 98 + return null; 99 + } 100 + 101 + const allPosts = new Map(); 102 + const now = new Date().toISOString(); 103 + 104 + // ── YouTube talks ──────────────────────────────────────────────── 105 + console.log('--- YouTube talks ---'); 106 + for (const q of [ 107 + { q: 'atmosphereconf', domain: 'youtube.com' }, 108 + { q: 'atmosphereconf', domain: 'youtu.be' }, 109 + { q: 'atmosphere talk', domain: 'youtube.com' }, 110 + { q: 'atmosphere conference talk', domain: 'youtu.be' }, 111 + ]) { 112 + try { 113 + const res = await agent.app.bsky.feed.searchPosts({ 114 + q: q.q, domain: q.domain, since: '2026-03-25T00:00:00Z', sort: 'top', limit: 50 115 + }); 116 + for (const p of (res.data?.posts || [])) { 117 + if (!allPosts.has(p.uri)) allPosts.set(p.uri, { post: p, type: 'video' }); 118 + } 119 + } catch {} 120 + await sleep(200); 121 + } 122 + console.log(` Found ${allPosts.size} YouTube-linked posts`); 123 + 124 + // ── More blog searches ─────────────────────────────────────────── 125 + console.log('\n--- More blog posts ---'); 126 + const blogQueries = [ 127 + { q: 'atmosphere', author: 'masnick.com', since: '2026-03-30T00:00:00Z' }, 128 + { q: 'atmosphere', author: 'mmccue.bsky.social', since: '2026-03-29T00:00:00Z' }, 129 + { q: 'atmosphere', author: 'cassidyjames.com', since: '2026-03-29T00:00:00Z' }, 130 + { q: 'atmosphere', author: 'katexcellence.io', since: '2026-03-29T00:00:00Z' }, 131 + { q: 'atmosphere', author: 'sooraj.dev', since: '2026-03-29T00:00:00Z' }, 132 + { q: 'atmosphere', author: 'werd.io', since: '2026-03-29T00:00:00Z' }, 133 + { q: 'atmosphere', author: 'bmann.ca', since: '2026-03-29T00:00:00Z' }, 134 + { q: 'atmosphere OR atmosphereconf', domain: 'pckt.blog', since: '2026-03-25T00:00:00Z' }, 135 + { q: 'atmosphere OR atmosphereconf', domain: 'brookie.pckt.blog', since: '2026-03-25T00:00:00Z' }, 136 + { q: 'atmosphere OR atmosphereconf', domain: 'connectedplaces.online', since: '2026-03-25T00:00:00Z' }, 137 + { q: 'atmosphereconf wrote', since: '2026-03-29T00:00:00Z' }, 138 + { q: 'atmosphereconf blog post', since: '2026-03-29T00:00:00Z' }, 139 + { q: 'atmosphere conference wrote about', since: '2026-03-29T00:00:00Z' }, 140 + ]; 141 + const beforeBlogs = allPosts.size; 142 + for (const bq of blogQueries) { 143 + try { 144 + const params = { q: bq.q, since: bq.since, sort: 'top', limit: 50 }; 145 + if (bq.author) params.author = bq.author; 146 + if (bq.domain) params.domain = bq.domain; 147 + const res = await agent.app.bsky.feed.searchPosts(params); 148 + for (const p of (res.data?.posts || [])) { 149 + if (!allPosts.has(p.uri)) { 150 + const links = extractLinks(p); 151 + const hasExternalLink = links.some(l => !l.includes('bsky.app')); 152 + allPosts.set(p.uri, { post: p, type: hasExternalLink ? 'blog' : 'post' }); 153 + } 154 + } 155 + } catch {} 156 + await sleep(200); 157 + } 158 + console.log(` Found ${allPosts.size - beforeBlogs} new blog-related posts`); 159 + 160 + // ── Photo posts ────────────────────────────────────────────────── 161 + console.log('\n--- Photo posts ---'); 162 + const photoQueries = [ 163 + '#atmosphereconf', 'atmosphereconf photo', 'atmosphereconf pic', 164 + 'atmosphereconf selfie', 'atmosphereconf group', 'atmosphere conf', 165 + ]; 166 + const beforePhotos = allPosts.size; 167 + for (const q of photoQueries) { 168 + try { 169 + const res = await agent.app.bsky.feed.searchPosts({ 170 + q, since: '2026-03-25T00:00:00Z', sort: 'top', limit: 100, 171 + }); 172 + for (const p of (res.data?.posts || [])) { 173 + if (!allPosts.has(p.uri) && hasImages(p)) { 174 + allPosts.set(p.uri, { post: p, type: 'photo' }); 175 + } 176 + } 177 + } catch {} 178 + await sleep(200); 179 + } 180 + console.log(` Found ${allPosts.size - beforePhotos} photo posts`); 181 + 182 + // ── Process all ────────────────────────────────────────────────── 183 + console.log(`\n--- Processing ${allPosts.size} total posts ---`); 184 + let counts = { blog: 0, video: 0, photo: 0, post: 0 }; 185 + 186 + const rows = []; 187 + for (const [uri, { post: p, type }] of allPosts) { 188 + const links = extractLinks(p); 189 + const externalUrl = links.find(l => !l.includes('bsky.app')) || null; 190 + const talkRkey = matchTalk(p, externalUrl); 191 + const talkUri = talkRkey ? (talksByRkey.get(talkRkey)?.uri || null) : null; 192 + const isPhoto = hasImages(p); 193 + 194 + // Refine type 195 + let contentType = type; 196 + if (contentType === 'post' && isPhoto) contentType = 'photo'; 197 + 198 + counts[contentType] = (counts[contentType] || 0) + 1; 199 + 200 + rows.push([ 201 + p.uri, talkUri, p.author.did, p.author.handle, 202 + p.record?.text, p.record?.createdAt, 203 + null, null, p.likeCount || 0, p.repostCount || 0, p.replyCount || 0, 204 + null, 'discussion', now, contentType, externalUrl, null, talkRkey, 205 + isPhoto ? 1 : 0, 206 + ]); 207 + } 208 + 209 + const batchInsert = db.transaction((items) => { for (const r of items) upsert.run(...r); }); 210 + batchInsert(rows); 211 + console.log(` Inserted: blog=${counts.blog}, video=${counts.video}, photo=${counts.photo}, post=${counts.post}`); 212 + 213 + // ── OG titles for new blogs ────────────────────────────────────── 214 + console.log('\n--- Fetching OG titles ---'); 215 + const needOg = db.prepare( 216 + "SELECT uri, external_url FROM mentions WHERE content_type = 'blog' AND external_url IS NOT NULL AND og_title IS NULL" 217 + ).all(); 218 + let ogCount = 0; 219 + const updateOg = db.prepare("UPDATE mentions SET og_title = ? WHERE uri = ?"); 220 + for (const row of needOg) { 221 + const title = await fetchOgTitle(row.external_url); 222 + if (title) { updateOg.run(title, row.uri); ogCount++; console.log(` ${row.external_url} → ${title}`); } 223 + await sleep(100); 224 + } 225 + console.log(` Fetched ${ogCount}/${needOg.length} OG titles`); 226 + 227 + // ── Profile backfill ───────────────────────────────────────────── 228 + console.log('\n--- Profile backfill ---'); 229 + const missing = db.prepare(` 230 + SELECT DISTINCT m.author_did FROM mentions m 231 + LEFT JOIN profiles p ON m.author_did = p.did WHERE p.did IS NULL 232 + `).all(); 233 + const profileUpsert = db.prepare( 234 + "INSERT OR REPLACE INTO profiles (did, handle, display_name, avatar_url, fetched_at) VALUES (?, ?, ?, ?, ?)" 235 + ); 236 + let pCount = 0; 237 + for (const { author_did: did } of missing) { 238 + try { 239 + const res = await fetch(`https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=${encodeURIComponent(did)}`); 240 + if (res.ok) { const d = await res.json(); profileUpsert.run(did, d.handle || null, d.displayName || null, d.avatar || null, now); pCount++; } 241 + } catch {} 242 + await sleep(50); 243 + } 244 + console.log(` New profiles: ${pCount}`); 245 + 246 + // ── Mark existing image posts ──────────────────────────────────── 247 + // We can't retroactively check images for existing posts without re-fetching, 248 + // but we've tagged all new ones. 249 + 250 + // Summary 251 + const stats = db.prepare("SELECT content_type, COUNT(*) as c FROM mentions GROUP BY content_type").all(); 252 + console.log('\n=== DONE ==='); 253 + for (const s of stats) console.log(` ${s.content_type}: ${s.c}`); 254 + console.log(` Total: ${db.prepare('SELECT COUNT(*) as c FROM mentions').get().c}`); 255 + db.close(); 256 + } 257 + 258 + main().catch(console.error);

Configure Feed

Configure Feed