feat: add transcription and talk index build pipelines

+531

2 changed files

expand all

scripts

transcribe.ts

+323

scripts/build-talk-index.ts

··· 1 + import "dotenv/config"; 2 + import * as fs from "fs"; 3 + import * as path from "path"; 4 + 5 + const REPO_DID = "did:plc:rbvrr34edl5ddpuwcubjiost"; 6 + const CONF_DID = "did:plc:3xewinw4wtimo2lqfy5fm5sw"; 7 + const PDS_HOST = "https://iameli.com"; 8 + const BSKY_HOST = "https://bsky.social"; 9 + const VOD_HOST = "https://vod-beta.stream.place"; 10 + const VIDEO_COLLECTION = "place.stream.video"; 11 + const EVENT_COLLECTION = "community.lexicon.calendar.event"; 12 + const DATA_DIR = path.resolve(__dirname, "../data"); 13 + const TRANSCRIPT_DIR = path.join(DATA_DIR, "transcripts"); 14 + 15 + interface VideoRecord { 16 + uri: string; 17 + cid: string; 18 + value: { 19 + $type: string; 20 + title: string; 21 + source: { 22 + ref: string; 23 + size: number; 24 + $type: string; 25 + mimeType: string; 26 + start?: number; 27 + end?: number; 28 + }; 29 + creator: string; 30 + duration: number; 31 + createdAt: string; 32 + }; 33 + } 34 + 35 + interface Speaker { 36 + id: string; // Bluesky handle 37 + name: string; 38 + } 39 + 40 + interface EventRecord { 41 + uri: string; 42 + cid: string; 43 + value: { 44 + $type: string; 45 + name: string; 46 + description: string; 47 + startsAt: string; 48 + endsAt: string; 49 + mode: string; 50 + status: string; 51 + createdAt: string; 52 + additionalData: { 53 + room?: string; 54 + type?: string; 55 + category?: string; 56 + sourceId?: string; 57 + speakers?: Speaker[]; 58 + vodAtUri?: string; 59 + isAtmosphereconf?: boolean; 60 + }; 61 + }; 62 + } 63 + 64 + interface TalkEntry { 65 + rkey: string; 66 + title: string; 67 + vodUri: string; 68 + vodCid: string; 69 + hlsUrl: string; 70 + durationMs: number; 71 + createdAt: string; 72 + // From schedule 73 + eventUri: string | null; 74 + description: string | null; 75 + speakers: Speaker[]; 76 + room: string | null; 77 + talkType: string | null; 78 + category: string | null; 79 + startsAt: string | null; 80 + endsAt: string | null; 81 + // Transcript 82 + transcriptFile: string | null; 83 + } 84 + 85 + async function fetchPaginatedRecords<T>( 86 + host: string, 87 + repo: string, 88 + collection: string, 89 + ): Promise<T[]> { 90 + const records: T[] = []; 91 + let cursor: string | undefined; 92 + 93 + do { 94 + const params = new URLSearchParams({ 95 + repo, 96 + collection, 97 + limit: "100", 98 + }); 99 + if (cursor) params.set("cursor", cursor); 100 + 101 + const url = `${host}/xrpc/com.atproto.repo.listRecords?${params}`; 102 + const res = await fetch(url); 103 + if (!res.ok) throw new Error(`Failed to fetch ${collection}: ${res.status}`); 104 + 105 + const data = await res.json(); 106 + records.push(...data.records); 107 + cursor = data.cursor; 108 + } while (cursor); 109 + 110 + return records; 111 + } 112 + 113 + /** 114 + * Normalize a title for fuzzy comparison: lowercase, strip punctuation, 115 + * remove common filler words, collapse whitespace. 116 + */ 117 + function normalizeTitle(title: string): string { 118 + return title 119 + .toLowerCase() 120 + .replace(/[^a-z0-9\s]/g, "") 121 + .replace( 122 + /\b(the|a|an|at|on|in|of|for|and|with|to|from|by|is|are|was|its)\b/g, 123 + "", 124 + ) 125 + .replace(/\s+/g, " ") 126 + .trim(); 127 + } 128 + 129 + /** 130 + * Score how similar two titles are by counting shared significant words. 131 + * Returns a value between 0 and 1. 132 + */ 133 + function titleSimilarity(a: string, b: string): number { 134 + const wordsA = new Set(normalizeTitle(a).split(" ").filter((w) => w.length > 2)); 135 + const wordsB = new Set(normalizeTitle(b).split(" ").filter((w) => w.length > 2)); 136 + if (wordsA.size === 0 || wordsB.size === 0) return 0; 137 + let shared = 0; 138 + for (const w of wordsA) { 139 + if (wordsB.has(w)) shared++; 140 + } 141 + return shared / Math.max(wordsA.size, wordsB.size); 142 + } 143 + 144 + /** 145 + * Try to match an unmatched VOD to a schedule event using fallback heuristics: 146 + * 1. Exact timestamp match (createdAt === startsAt) 147 + * 2. Title similarity (>= 0.4 threshold) 148 + * Returns the best matching event or null. 149 + */ 150 + function fallbackMatch( 151 + vod: VideoRecord, 152 + candidates: EventRecord[], 153 + ): EventRecord | null { 154 + // 1. Exact timestamp match 155 + const vodTime = new Date(vod.value.createdAt).getTime(); 156 + for (const event of candidates) { 157 + const eventTime = new Date(event.value.startsAt).getTime(); 158 + if (Math.abs(vodTime - eventTime) < 60_000) { 159 + return event; 160 + } 161 + } 162 + 163 + // 2. Title similarity — require higher threshold for short titles 164 + // to avoid false matches like "closing remarks" → "Opening Remarks" 165 + let bestEvent: EventRecord | null = null; 166 + let bestScore = 0; 167 + for (const event of candidates) { 168 + const score = titleSimilarity(vod.value.title, event.value.name); 169 + if (score > bestScore) { 170 + bestScore = score; 171 + bestEvent = event; 172 + } 173 + } 174 + const normalizedWords = normalizeTitle(vod.value.title).split(" ").filter((w) => w.length > 2); 175 + const threshold = normalizedWords.length <= 3 ? 0.7 : 0.35; 176 + if (bestScore >= threshold && bestEvent) { 177 + return bestEvent; 178 + } 179 + 180 + return null; 181 + } 182 + 183 + function getRkey(uri: string): string { 184 + return uri.split("/").pop()!; 185 + } 186 + 187 + function getPlaylistUrl(uri: string): string { 188 + return `${VOD_HOST}/xrpc/place.stream.playback.getVideoPlaylist?uri=${encodeURIComponent(uri)}`; 189 + } 190 + 191 + // Duration in the video records is in nanoseconds. Convert to ms. 192 + function nsToMs(ns: number): number { 193 + return Math.round(ns / 1_000_000); 194 + } 195 + 196 + async function main() { 197 + console.log("Fetching video records..."); 198 + const videoRecords = await fetchPaginatedRecords<VideoRecord>( 199 + PDS_HOST, 200 + REPO_DID, 201 + VIDEO_COLLECTION, 202 + ); 203 + console.log(`Found ${videoRecords.length} video records`); 204 + 205 + // Filter to individual talk VODs (exclude room streams created by repo owner) 206 + const talkVods = videoRecords.filter((r) => r.value.creator !== REPO_DID); 207 + console.log(`Filtered to ${talkVods.length} individual talk VODs`); 208 + 209 + console.log("\nFetching schedule events..."); 210 + const eventRecords = await fetchPaginatedRecords<EventRecord>( 211 + BSKY_HOST, 212 + CONF_DID, 213 + EVENT_COLLECTION, 214 + ); 215 + console.log(`Found ${eventRecords.length} schedule events`); 216 + 217 + // Build a lookup: vodAtUri -> event record 218 + const eventByVodUri = new Map<string, EventRecord>(); 219 + const matchedEventUris = new Set<string>(); 220 + for (const event of eventRecords) { 221 + const vodUri = event.value.additionalData?.vodAtUri; 222 + if (vodUri) { 223 + eventByVodUri.set(vodUri, event); 224 + } 225 + } 226 + console.log(`${eventByVodUri.size} events have VOD links`); 227 + 228 + // First pass: match by vodAtUri 229 + const unmatchedVods: VideoRecord[] = []; 230 + const vodToEvent = new Map<string, EventRecord>(); 231 + 232 + for (const vod of talkVods) { 233 + const event = eventByVodUri.get(vod.uri); 234 + if (event) { 235 + vodToEvent.set(vod.uri, event); 236 + matchedEventUris.add(event.uri); 237 + } else { 238 + unmatchedVods.push(vod); 239 + } 240 + } 241 + console.log(`First pass (vodAtUri): ${vodToEvent.size} matched, ${unmatchedVods.length} unmatched`); 242 + 243 + // Second pass: fallback matching for unmatched VODs 244 + // Candidates are events not already matched by vodAtUri 245 + const fallbackCandidates = eventRecords.filter( 246 + (e) => !matchedEventUris.has(e.uri), 247 + ); 248 + let fallbackMatched = 0; 249 + 250 + for (const vod of [...unmatchedVods]) { 251 + const event = fallbackMatch(vod, fallbackCandidates); 252 + if (event) { 253 + vodToEvent.set(vod.uri, event); 254 + matchedEventUris.add(event.uri); 255 + // Remove from candidates so it can't double-match 256 + const idx = fallbackCandidates.indexOf(event); 257 + if (idx >= 0) fallbackCandidates.splice(idx, 1); 258 + // Remove from unmatched 259 + unmatchedVods.splice(unmatchedVods.indexOf(vod), 1); 260 + fallbackMatched++; 261 + console.log(` [fallback] "${vod.value.title}" → "${event.value.name}"`); 262 + } 263 + } 264 + console.log(`Second pass (fallback): ${fallbackMatched} matched, ${unmatchedVods.length} still unmatched`); 265 + 266 + for (const vod of unmatchedVods) { 267 + console.log(` [no match] ${vod.value.title}`); 268 + } 269 + 270 + // Build the index 271 + const talks: TalkEntry[] = []; 272 + let matched = vodToEvent.size; 273 + let unmatched = talkVods.length - matched; 274 + 275 + for (const vod of talkVods) { 276 + const rkey = getRkey(vod.uri); 277 + const event = vodToEvent.get(vod.uri); 278 + 279 + const transcriptFile = `transcripts/${rkey}.json`; 280 + const hasTranscript = fs.existsSync(path.join(DATA_DIR, transcriptFile)); 281 + 282 + talks.push({ 283 + rkey, 284 + title: vod.value.title, 285 + vodUri: vod.uri, 286 + vodCid: vod.cid, 287 + hlsUrl: getPlaylistUrl(vod.uri), 288 + durationMs: nsToMs(vod.value.duration), 289 + createdAt: vod.value.createdAt, 290 + eventUri: event?.uri ?? null, 291 + description: event?.value.description ?? null, 292 + speakers: event?.value.additionalData?.speakers ?? [], 293 + room: event?.value.additionalData?.room ?? null, 294 + talkType: event?.value.additionalData?.type ?? null, 295 + category: event?.value.additionalData?.category ?? null, 296 + startsAt: event?.value.startsAt ?? null, 297 + endsAt: event?.value.endsAt ?? null, 298 + transcriptFile: hasTranscript ? transcriptFile : null, 299 + }); 300 + } 301 + 302 + // Sort by schedule time (talks with schedule first, then by time) 303 + talks.sort((a, b) => { 304 + if (a.startsAt && b.startsAt) return a.startsAt.localeCompare(b.startsAt); 305 + if (a.startsAt) return -1; 306 + if (b.startsAt) return 1; 307 + return a.createdAt.localeCompare(b.createdAt); 308 + }); 309 + 310 + // Write output 311 + const outputPath = path.join(DATA_DIR, "talks.json"); 312 + fs.mkdirSync(DATA_DIR, { recursive: true }); 313 + fs.writeFileSync(outputPath, JSON.stringify(talks, null, 2)); 314 + 315 + console.log(`\n=== Summary ===`); 316 + console.log(`Total talk VODs: ${talkVods.length}`); 317 + console.log(`Matched to schedule: ${matched}`); 318 + console.log(`Unmatched: ${unmatched}`); 319 + console.log(`With transcripts: ${talks.filter((t) => t.transcriptFile).length}`); 320 + console.log(`\nWritten to ${outputPath}`); 321 + } 322 + 323 + main().catch(console.error);

+208

scripts/transcribe.ts

··· 1 + import "dotenv/config"; 2 + import { AssemblyAI } from "assemblyai"; 3 + import { execFileSync } from "child_process"; 4 + import * as fs from "fs"; 5 + import * as path from "path"; 6 + 7 + const REPO_DID = "did:plc:rbvrr34edl5ddpuwcubjiost"; 8 + const PDS_HOST = "https://iameli.com"; 9 + const VOD_HOST = "https://vod-beta.stream.place"; 10 + const COLLECTION = "place.stream.video"; 11 + const TRANSCRIPT_DIR = path.resolve(__dirname, "../data/transcripts"); 12 + const BATCH_SIZE = 10; 13 + 14 + interface VideoRecord { 15 + uri: string; 16 + cid: string; 17 + value: { 18 + $type: string; 19 + title: string; 20 + source: { 21 + ref: string; 22 + size: number; 23 + $type: string; 24 + mimeType: string; 25 + start?: number; 26 + end?: number; 27 + }; 28 + creator: string; 29 + duration: number; 30 + createdAt: string; 31 + }; 32 + } 33 + 34 + interface ListRecordsResponse { 35 + records: VideoRecord[]; 36 + cursor?: string; 37 + } 38 + 39 + async function fetchAllVideoRecords(): Promise<VideoRecord[]> { 40 + const records: VideoRecord[] = []; 41 + let cursor: string | undefined; 42 + 43 + do { 44 + const params = new URLSearchParams({ 45 + repo: REPO_DID, 46 + collection: COLLECTION, 47 + limit: "100", 48 + }); 49 + if (cursor) params.set("cursor", cursor); 50 + 51 + const url = `${PDS_HOST}/xrpc/com.atproto.repo.listRecords?${params}`; 52 + const res = await fetch(url); 53 + if (!res.ok) throw new Error(`Failed to fetch records: ${res.status}`); 54 + 55 + const data: ListRecordsResponse = await res.json(); 56 + records.push(...data.records); 57 + cursor = data.cursor; 58 + } while (cursor); 59 + 60 + return records; 61 + } 62 + 63 + function isIndividualTalkVOD(record: VideoRecord): boolean { 64 + // Full-day room streams are created by the repo owner (Streamplace itself). 65 + // Individual talk VODs are created by different DIDs (room-specific stream accounts). 66 + if (record.value.creator === REPO_DID) return false; 67 + return true; 68 + } 69 + 70 + function getRkey(uri: string): string { 71 + return uri.split("/").pop()!; 72 + } 73 + 74 + function getPlaylistUrl(uri: string): string { 75 + return `${VOD_HOST}/xrpc/place.stream.playback.getVideoPlaylist?uri=${encodeURIComponent(uri)}`; 76 + } 77 + 78 + async function extractAudio( 79 + playlistUrl: string, 80 + outputPath: string, 81 + ): Promise<void> { 82 + console.log(` Extracting audio to ${path.basename(outputPath)}...`); 83 + execFileSync( 84 + "ffmpeg", 85 + ["-y", "-i", playlistUrl, "-vn", "-q:a", "2", outputPath], 86 + { timeout: 600_000, stdio: ["pipe", "pipe", "pipe"] }, 87 + ); 88 + } 89 + 90 + async function transcribeTalk( 91 + client: AssemblyAI, 92 + record: VideoRecord, 93 + ): Promise<void> { 94 + const rkey = getRkey(record.uri); 95 + const outputPath = path.join(TRANSCRIPT_DIR, `${rkey}.json`); 96 + 97 + // Skip if already transcribed 98 + if (fs.existsSync(outputPath)) { 99 + console.log(` [skip] ${record.value.title} — already transcribed`); 100 + return; 101 + } 102 + 103 + const mp3Path = `/tmp/${rkey}.mp3`; 104 + const playlistUrl = getPlaylistUrl(record.uri); 105 + 106 + try { 107 + // Extract audio 108 + await extractAudio(playlistUrl, mp3Path); 109 + 110 + // Transcribe with AssemblyAI 111 + console.log(` Transcribing: ${record.value.title}...`); 112 + const transcript = await client.transcripts.transcribe({ 113 + audio: mp3Path, 114 + speaker_labels: true, 115 + speech_models: ["universal-3-pro", "universal-2"], 116 + } as any); 117 + 118 + if (transcript.status === "error") { 119 + console.error(` [error] ${record.value.title}: ${transcript.error}`); 120 + return; 121 + } 122 + 123 + // Save result 124 + const result = { 125 + uri: record.uri, 126 + cid: record.cid, 127 + title: record.value.title, 128 + creator: record.value.creator, 129 + duration: record.value.duration, 130 + createdAt: record.value.createdAt, 131 + transcription: { 132 + id: transcript.id, 133 + status: transcript.status, 134 + text: transcript.text, 135 + utterances: transcript.utterances, 136 + words: transcript.words, 137 + audio_duration: transcript.audio_duration, 138 + }, 139 + }; 140 + 141 + fs.writeFileSync(outputPath, JSON.stringify(result, null, 2)); 142 + console.log( 143 + ` [done] ${record.value.title} (${transcript.audio_duration}s)`, 144 + ); 145 + } finally { 146 + // Clean up mp3 147 + if (fs.existsSync(mp3Path)) fs.unlinkSync(mp3Path); 148 + } 149 + } 150 + 151 + async function processBatch( 152 + client: AssemblyAI, 153 + records: VideoRecord[], 154 + ): Promise<void> { 155 + await Promise.all(records.map((r) => transcribeTalk(client, r))); 156 + } 157 + 158 + async function main() { 159 + const apiKey = process.env.ASSEMBLYAI_API_KEY; 160 + if (!apiKey) { 161 + console.error("ASSEMBLYAI_API_KEY not set"); 162 + process.exit(1); 163 + } 164 + 165 + const client = new AssemblyAI({ apiKey }); 166 + 167 + // Ensure output directory exists 168 + fs.mkdirSync(TRANSCRIPT_DIR, { recursive: true }); 169 + 170 + // Fetch all video records 171 + console.log("Fetching video records..."); 172 + const allRecords = await fetchAllVideoRecords(); 173 + console.log(`Found ${allRecords.length} total video records`); 174 + 175 + // Filter to individual talk VODs 176 + const talks = allRecords.filter(isIndividualTalkVOD); 177 + console.log(`Filtered to ${talks.length} individual talk VODs`); 178 + console.log( 179 + `Skipped ${allRecords.length - talks.length} full-day room streams`, 180 + ); 181 + 182 + // Check which ones are already done 183 + const remaining = talks.filter((t) => { 184 + const rkey = getRkey(t.uri); 185 + return !fs.existsSync(path.join(TRANSCRIPT_DIR, `${rkey}.json`)); 186 + }); 187 + console.log( 188 + `${talks.length - remaining.length} already transcribed, ${remaining.length} remaining\n`, 189 + ); 190 + 191 + // Process in batches of BATCH_SIZE 192 + for (let i = 0; i < remaining.length; i += BATCH_SIZE) { 193 + const batch = remaining.slice(i, i + BATCH_SIZE); 194 + const batchNum = Math.floor(i / BATCH_SIZE) + 1; 195 + const totalBatches = Math.ceil(remaining.length / BATCH_SIZE); 196 + console.log( 197 + `\n=== Batch ${batchNum}/${totalBatches} (${batch.length} talks) ===`, 198 + ); 199 + batch.forEach((r) => console.log(` - ${r.value.title}`)); 200 + console.log(); 201 + 202 + await processBatch(client, batch); 203 + } 204 + 205 + console.log("\nDone! All talks transcribed."); 206 + } 207 + 208 + main().catch(console.error);

Configure Feed

Configure Feed