Ionosphere.tv
feat: fetch more blog posts — leaflet, pckt, myhub, connectedplaces
Found Catherine Tait's ATmosphere reflections on Leaflet, plus ROOST
writeup, The Joy of AtmosphereConf, ATProto Newbie thoughts, and more.
47 blog posts total with 25 OG titles fetched.
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+143
+143
scripts/fetch-blogs-extra.mjs
+143
scripts/fetch-blogs-extra.mjs
···
1
+
/**
2
+
* Fetch additional blog posts from leaflet, pckt, myhub, and other sources.
3
+
* Also fetches OG titles for new blog entries.
4
+
*/
5
+
import { createRequire } from 'module';
6
+
const require = createRequire(
7
+
new URL('../apps/ionosphere-appview/package.json', import.meta.url).pathname
8
+
);
9
+
const { BskyAgent } = require('@atproto/api');
10
+
const Database = require('better-sqlite3');
11
+
import { fileURLToPath } from 'url';
12
+
import { dirname, join } from 'path';
13
+
14
+
const __dirname = dirname(fileURLToPath(import.meta.url));
15
+
const DB_PATH = join(__dirname, '..', 'apps', 'data', 'ionosphere.sqlite');
16
+
17
+
const agent = new BskyAgent({ service: 'https://bsky.social' });
18
+
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
19
+
20
+
async function fetchOgTitle(url) {
21
+
try {
22
+
const controller = new AbortController();
23
+
const timeout = setTimeout(() => controller.abort(), 5000);
24
+
const res = await globalThis.fetch(url, {
25
+
signal: controller.signal,
26
+
headers: { 'User-Agent': 'ionosphere.tv/1.0' },
27
+
redirect: 'follow',
28
+
});
29
+
clearTimeout(timeout);
30
+
if (!res.ok) return null;
31
+
const html = await res.text();
32
+
const ogMatch = html.match(/<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i)
33
+
|| html.match(/<meta[^>]+content=["']([^"']+)["'][^>]+property=["']og:title["']/i);
34
+
if (ogMatch) return ogMatch[1];
35
+
const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/i);
36
+
return titleMatch ? titleMatch[1].trim() : null;
37
+
} catch { return null; }
38
+
}
39
+
40
+
async function main() {
41
+
console.log('=== Fetch Extra Blog Posts ===\n');
42
+
await agent.login({ identifier: 'ionosphere.tv', password: process.env.BOT_PASSWORD });
43
+
44
+
const db = new Database(DB_PATH);
45
+
const now = new Date().toISOString();
46
+
47
+
const upsert = db.prepare(`
48
+
INSERT INTO mentions (uri, talk_uri, author_did, author_handle, text, created_at,
49
+
talk_offset_ms, byte_position, likes, reposts, replies, parent_uri,
50
+
mention_type, indexed_at, content_type, external_url, og_title, talk_rkey)
51
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
52
+
ON CONFLICT(uri) DO UPDATE SET
53
+
likes=excluded.likes, reposts=excluded.reposts,
54
+
content_type=CASE WHEN excluded.content_type = 'blog' THEN 'blog' ELSE mentions.content_type END,
55
+
external_url=COALESCE(excluded.external_url, mentions.external_url),
56
+
og_title=COALESCE(excluded.og_title, mentions.og_title),
57
+
indexed_at=excluded.indexed_at
58
+
`);
59
+
60
+
const queries = [
61
+
{ q: 'atmosphere', domain: 'leaflet.pub' },
62
+
{ q: 'atmosphereconf', domain: 'leaflet.pub' },
63
+
{ q: 'atmosphere', domain: 'pckt.blog' },
64
+
{ q: 'atmosphereconf', domain: 'pckt.blog' },
65
+
{ q: 'atmosphere', domain: 'brookie.pckt.blog' },
66
+
{ q: 'atmosphere', domain: 'experiments.myhub.ai' },
67
+
{ q: 'atmosphere', domain: 'connectedplaces.online' },
68
+
{ q: 'atmosphere', domain: 'masnick.com' },
69
+
{ q: 'atmosphere', domain: 'cassidyjames.com' },
70
+
{ q: 'atmosphere', domain: 'brittanyellich.com' },
71
+
{ q: 'atmosphere', domain: 'gui.do' },
72
+
{ q: 'atmosphere', domain: 'sooraj.dev' },
73
+
// General blog searches
74
+
{ q: 'atmosphereconf wrote blog' },
75
+
{ q: 'atmosphereconf wrote about' },
76
+
{ q: 'atmosphereconf blog post' },
77
+
{ q: 'atmosphereconf reflection' },
78
+
{ q: 'atmosphereconf experience wrote' },
79
+
];
80
+
81
+
let count = 0;
82
+
for (const bq of queries) {
83
+
try {
84
+
const params = { q: bq.q, since: '2026-03-25T00:00:00Z', sort: 'top', limit: 50 };
85
+
if (bq.domain) params.domain = bq.domain;
86
+
const res = await agent.app.bsky.feed.searchPosts(params);
87
+
const posts = res.data?.posts || [];
88
+
if (posts.length > 0) console.log(` "${bq.q}"${bq.domain ? ' domain:' + bq.domain : ''}: ${posts.length}`);
89
+
90
+
for (const p of posts) {
91
+
let externalUrl = p.embed?.external?.uri || null;
92
+
if (!externalUrl) {
93
+
const links = (p.record?.facets || []).flatMap(f => f.features || []).filter(f => f.uri).map(f => f.uri);
94
+
externalUrl = links.find(l => !l.includes('bsky.app')) || null;
95
+
}
96
+
97
+
upsert.run(
98
+
p.uri, null, p.author.did, p.author.handle,
99
+
p.record?.text, p.record?.createdAt,
100
+
null, null, p.likeCount || 0, p.repostCount || 0, p.replyCount || 0,
101
+
null, 'discussion', now, 'blog', externalUrl, null, null
102
+
);
103
+
count++;
104
+
}
105
+
} catch (e) {
106
+
console.error(` Error: ${e.message}`);
107
+
}
108
+
await sleep(200);
109
+
}
110
+
console.log(`\nUpserted ${count} blog rows`);
111
+
112
+
// Fetch OG titles
113
+
console.log('\n--- OG titles ---');
114
+
const needOg = db.prepare(
115
+
"SELECT uri, external_url FROM mentions WHERE content_type = 'blog' AND external_url IS NOT NULL AND og_title IS NULL"
116
+
).all();
117
+
const updateOg = db.prepare('UPDATE mentions SET og_title = ? WHERE uri = ?');
118
+
let ogCount = 0;
119
+
for (const row of needOg) {
120
+
const title = await fetchOgTitle(row.external_url);
121
+
if (title) { updateOg.run(title, row.uri); ogCount++; console.log(` ${row.external_url} → ${title}`); }
122
+
await sleep(100);
123
+
}
124
+
console.log(`Fetched ${ogCount}/${needOg.length} OG titles`);
125
+
126
+
// Profile backfill
127
+
const missing = db.prepare('SELECT DISTINCT m.author_did FROM mentions m LEFT JOIN profiles p ON m.author_did = p.did WHERE p.did IS NULL').all();
128
+
const profileUpsert = db.prepare('INSERT OR REPLACE INTO profiles (did, handle, display_name, avatar_url, fetched_at) VALUES (?, ?, ?, ?, ?)');
129
+
let pCount = 0;
130
+
for (const { author_did: did } of missing) {
131
+
try {
132
+
const r = await globalThis.fetch(`https://public.api.bsky.app/xrpc/app.bsky.actor.getProfile?actor=${encodeURIComponent(did)}`);
133
+
if (r.ok) { const d = await r.json(); profileUpsert.run(did, d.handle || null, d.displayName || null, d.avatar || null, now); pCount++; }
134
+
} catch {}
135
+
await sleep(50);
136
+
}
137
+
if (pCount) console.log(`New profiles: ${pCount}`);
138
+
139
+
console.log('\nBlogs total:', db.prepare("SELECT COUNT(*) as c FROM mentions WHERE content_type = 'blog'").get().c);
140
+
db.close();
141
+
}
142
+
143
+
main().catch(console.error);