feat: split segmentation into textSpan + temporal records · blaine.bsky.social/ionosphere@26a4761

+25 -8

apps/ionosphere-appview/src/__tests__/layers-pub.test.ts

··· 27 27 expect(expression.createdAt).toBeDefined(); 28 28 }); 29 29 30 - it('produces a segmentation record with word tokens', async () => { 30 + it('produces a segmentation record with textSpan-only tokens', async () => { 31 31 const { segmentation } = await transcriptToLayersPub(transcript, did, talkRkey); 32 32 expect(segmentation.$type).toBe('pub.layers.segmentation.segmentation'); 33 33 expect(segmentation.expression).toBe( ··· 39 39 expect(tok.kind).toBe('word'); 40 40 expect(tok.tokens).toHaveLength(4); 41 41 42 - // Check first token 42 + // Check first token — textSpan only, no temporalSpan 43 43 expect(tok.tokens[0].tokenIndex).toBe(0); 44 44 expect(tok.tokens[0].text).toBe('Hello'); 45 45 expect(tok.tokens[0].textSpan.byteStart).toBe(0); 46 46 expect(tok.tokens[0].textSpan.byteEnd).toBe(5); 47 - expect(tok.tokens[0].temporalSpan.start).toBe(1000); 48 - expect(tok.tokens[0].temporalSpan.ending).toBe(1200); 49 47 50 - // Check third token (after gap) — byte offsets and temporal span 48 + // Check third token (after gap) — byte offsets 51 49 expect(tok.tokens[2].text).toBe('foo'); 52 50 expect(tok.tokens[2].textSpan.byteStart).toBe(12); // "Hello world " = 12 bytes 53 51 expect(tok.tokens[2].textSpan.byteEnd).toBe(15); 52 + }); 53 + 54 + it('produces a separate temporal segmentation record', async () => { 55 + const { temporal } = await transcriptToLayersPub(transcript, did, talkRkey); 56 + expect(temporal.$type).toBe('pub.layers.segmentation.segmentation'); 57 + expect(temporal.tokenizations).toHaveLength(1); 58 + 59 + const tok = temporal.tokenizations[0]; 60 + expect(tok.kind).toBe('word-temporal'); 61 + expect(tok.tokens).toHaveLength(4); 62 + 63 + // First token temporal span 64 + expect(tok.tokens[0].tokenIndex).toBe(0); 65 + expect(tok.tokens[0].temporalSpan.start).toBe(1000); 66 + expect(tok.tokens[0].temporalSpan.ending).toBe(1200); 67 + 68 + // Third token (after gap) 54 69 expect(tok.tokens[2].temporalSpan.start).toBe(1600); // 1000+200+300+100gap 55 70 expect(tok.tokens[2].temporalSpan.ending).toBe(1750); 56 71 }); ··· 156 171 talkUri: `at://${did}/tv.ionosphere.talk/${rkey}`, 157 172 }; 158 173 159 - const { expression, segmentation } = await transcriptToLayersPub(transcriptRecord, did, rkey); 174 + const { expression, segmentation, temporal } = await transcriptToLayersPub(transcriptRecord, did, rkey); 160 175 const expressionUri = `at://${did}/pub.layers.expression.expression/${rkey}-expression`; 161 176 const layers = await nlpToAnnotationLayers(nlpData, did, rkey, expressionUri); 162 177 163 - // Verify all 6 records have correct $type 178 + // Verify all 7 records have correct $type 164 179 expect(expression.$type).toBe('pub.layers.expression.expression'); 165 180 expect(segmentation.$type).toBe('pub.layers.segmentation.segmentation'); 181 + expect(temporal.$type).toBe('pub.layers.segmentation.segmentation'); 166 182 expect(layers.sentences.$type).toBe('pub.layers.annotation.annotationLayer'); 167 183 expect(layers.paragraphs.$type).toBe('pub.layers.annotation.annotationLayer'); 168 184 expect(layers.entities.$type).toBe('pub.layers.annotation.annotationLayer'); ··· 170 186 171 187 // Verify real data produces non-trivial results 172 188 expect(segmentation.tokenizations[0].tokens.length).toBeGreaterThan(100); 189 + expect(temporal.tokenizations[0].tokens.length).toBeGreaterThan(100); 173 190 expect(layers.sentences.annotations.length).toBeGreaterThan(10); 174 191 expect(layers.entities.annotations.length).toBeGreaterThan(10); 175 192 }); ··· 215 232 const { expression, segmentation } = await transcriptToLayersPub(transcriptRecord, did, rkey); 216 233 const expressionUri = `at://${did}/pub.layers.expression.expression/${rkey}-expression`; 217 234 const layers = await nlpToAnnotationLayers(nlpData, did, rkey, expressionUri); 218 - const lensDoc = await layersPubToDocument(expression, segmentation, layers); 235 + const lensDoc = await layersPubToDocument(expression, segmentation, layers, compact); 219 236 220 237 // Compare text 221 238 expect(lensDoc.text).toBe(directDoc.text);

+14 -7

apps/ionosphere-appview/src/layers-indexer.ts

··· 191 191 topics: annotationLayers.topics ?? emptyLayer("span", "topic-segment"), 192 192 }; 193 193 194 - // 5. Run Lens 3 194 + // 5. Look up the transcript for timing data + talk_uri 195 + const transcript = db 196 + .prepare("SELECT * FROM transcripts WHERE uri = ?") 197 + .get(expr.transcript_uri) as any; 198 + if (!transcript) return; 199 + 200 + // 6. Run Lens 3 with compact transcript for timestamp facets 201 + const compact = { 202 + text: transcript.text, 203 + startMs: transcript.start_ms, 204 + timings: JSON.parse(transcript.timings), 205 + }; 206 + 195 207 const document = await layersPubToDocument( 196 208 expressionRecord, 197 209 segmentationRecord, 198 210 fullLayers, 211 + compact, 199 212 ); 200 - 201 - // 6. Find the talk_uri from the transcript table 202 - const transcript = db 203 - .prepare("SELECT talk_uri FROM transcripts WHERE uri = ?") 204 - .get(expr.transcript_uri) as any; 205 - if (!transcript) return; 206 213 207 214 // 7. Update the talk's document field 208 215 db.prepare("UPDATE talks SET document = ? WHERE uri = ?").run(

+3 -2

apps/ionosphere-appview/src/publish.ts

··· 214 214 talkUri: `at://${did}/tv.ionosphere.talk/${talk.rkey}`, 215 215 }; 216 216 217 - const { expression, segmentation } = await transcriptToLayersPub(transcriptRecord, did, talk.rkey); 217 + const { expression, segmentation, temporal } = await transcriptToLayersPub(transcriptRecord, did, talk.rkey); 218 218 const expressionUri = `at://${did}/pub.layers.expression.expression/${talk.rkey}-expression`; 219 219 const layers = await nlpToAnnotationLayers(nlpData, did, talk.rkey, expressionUri); 220 220 221 221 await Promise.all([ 222 222 pds.putRecord("pub.layers.expression.expression", `${talk.rkey}-expression`, expression), 223 223 pds.putRecord("pub.layers.segmentation.segmentation", `${talk.rkey}-segmentation`, segmentation), 224 + pds.putRecord("pub.layers.segmentation.segmentation", `${talk.rkey}-temporal`, temporal), 224 225 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-sentences`, layers.sentences), 225 226 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-paragraphs`, layers.paragraphs), 226 227 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-entities`, layers.entities), 227 228 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-topics`, layers.topics), 228 229 ]); 229 230 230 - console.log(` layers.pub: ${talk.rkey} (6 records)`); 231 + console.log(` layers.pub: ${talk.rkey} (7 records)`); 231 232 layersCount++; 232 233 } 233 234 console.log(`Published layers.pub records for ${layersCount} talks.`);

+81 -31

formats/tv.ionosphere/ts/layers-pub.ts

··· 16 16 * and indexOf with searchFrom for word position tracking. 17 17 */ 18 18 19 - import type { Document, DocumentFacet } from './transcript-encoding.js'; 19 + import type { Document, DocumentFacet, CompactTranscript } from './transcript-encoding.js'; 20 20 21 21 export interface TranscriptRecord { 22 22 $type: string; ··· 50 50 ending: number; 51 51 } 52 52 53 - export interface Token { 53 + export interface TextToken { 54 54 tokenIndex: number; 55 55 text: string; 56 56 textSpan: TokenSpan; 57 + } 58 + 59 + export interface TemporalToken { 60 + tokenIndex: number; 57 61 temporalSpan: TemporalSpan; 58 62 } 59 63 60 - export interface Tokenization { 64 + export interface TextTokenization { 61 65 kind: string; 62 - tokens: Token[]; 66 + tokens: TextToken[]; 67 + } 68 + 69 + export interface TemporalTokenization { 70 + kind: string; 71 + tokens: TemporalToken[]; 63 72 } 64 73 65 74 export interface SegmentationRecord { 66 75 $type: 'pub.layers.segmentation.segmentation'; 67 76 expression: string; 68 - tokenizations: Tokenization[]; 77 + tokenizations: TextTokenization[]; 78 + createdAt: string; 79 + } 80 + 81 + export interface TemporalSegmentationRecord { 82 + $type: 'pub.layers.segmentation.segmentation'; 83 + expression: string; 84 + tokenizations: TemporalTokenization[]; 69 85 createdAt: string; 70 86 } 71 87 72 88 export interface LayersPubResult { 73 89 expression: ExpressionRecord; 74 90 segmentation: SegmentationRecord; 91 + temporal: TemporalSegmentationRecord; 75 92 } 76 93 77 94 /** ··· 107 124 createdAt: now, 108 125 }; 109 126 110 - // Build segmentation record using timings replay algorithm 127 + // Build segmentation records using timings replay algorithm 111 128 // (matches decodeToDocument in transcript-encoding.ts) 129 + // Two records: textSpan-only (word boundaries) and temporalSpan-only (timing) 112 130 const encoder = new TextEncoder(); 113 131 const words = transcript.text.split(/\s+/).filter((w) => w.length > 0); 114 - const tokens: Token[] = []; 132 + const textTokens: TextToken[] = []; 133 + const temporalTokens: TemporalToken[] = []; 115 134 116 135 let cursor = transcript.startMs; // ms 117 136 let wordIndex = 0; ··· 132 151 transcript.text.slice(0, idx + word.length), 133 152 ).length; 134 153 135 - tokens.push({ 154 + textTokens.push({ 136 155 tokenIndex: wordIndex, 137 156 text: word, 138 157 textSpan: { byteStart, byteEnd }, 158 + }); 159 + 160 + temporalTokens.push({ 161 + tokenIndex: wordIndex, 139 162 temporalSpan: { start: cursor, ending: cursor + value }, 140 163 }); 141 164 ··· 147 170 } 148 171 } 149 172 173 + const expressionUri = `at://${did}/pub.layers.expression.expression/${talkRkey}-expression`; 174 + 150 175 const segmentation: SegmentationRecord = { 151 176 $type: 'pub.layers.segmentation.segmentation', 152 - expression: `at://${did}/pub.layers.expression.expression/${talkRkey}-expression`, 153 - tokenizations: [ 154 - { 155 - kind: 'word', 156 - tokens, 157 - }, 158 - ], 177 + expression: expressionUri, 178 + tokenizations: [{ kind: 'word', tokens: textTokens }], 159 179 createdAt: now, 160 180 }; 161 181 162 - return { expression, segmentation }; 182 + const temporal: TemporalSegmentationRecord = { 183 + $type: 'pub.layers.segmentation.segmentation', 184 + expression: expressionUri, 185 + tokenizations: [{ kind: 'word-temporal', tokens: temporalTokens }], 186 + createdAt: now, 187 + }; 188 + 189 + return { expression, segmentation, temporal }; 163 190 } 164 191 165 192 /** ··· 305 332 * This is the materialized view builder — used by the appview indexer to 306 333 * rebuild the talk document when layers.pub records arrive via Jetstream. 307 334 * 335 + * Timestamp facets are generated from the compact transcript (text + startMs 336 + * + timings), not from segmentation temporalSpans. This allows the 337 + * segmentation record to remain small (textSpan only) while keeping temporal 338 + * data available separately. 339 + * 308 340 * Round-trip property: 309 341 * transcriptToLayersPub + nlpToAnnotationLayers + layersPubToDocument 310 342 * should produce the SAME document as decodeToDocumentWithStructure. ··· 313 345 expression: ExpressionRecord, 314 346 segmentation: SegmentationRecord, 315 347 annotationLayers: AnnotationLayersResult, 348 + compact?: CompactTranscript, 316 349 ): Promise<Document> { 317 350 const facets: DocumentFacet[] = []; 318 351 319 - // 1. Timestamp facets from segmentation tokens 320 - for (const token of segmentation.tokenizations[0].tokens) { 321 - facets.push({ 322 - index: { 323 - byteStart: token.textSpan.byteStart, 324 - byteEnd: token.textSpan.byteEnd, 325 - }, 326 - features: [ 327 - { 328 - $type: 'tv.ionosphere.facet#timestamp', 329 - startTime: token.temporalSpan.start * 1_000_000, // ms → ns 330 - endTime: token.temporalSpan.ending * 1_000_000, 331 - }, 332 - ], 333 - }); 352 + // 1. Timestamp facets from compact transcript timings 353 + // Uses the same replay algorithm as decodeToDocument() — the segmentation 354 + // provides byte offsets, the compact transcript provides timing. 355 + if (compact) { 356 + const tokens = segmentation.tokenizations[0].tokens; 357 + let cursor = compact.startMs; 358 + let tokenIdx = 0; 359 + 360 + for (const value of compact.timings) { 361 + if (value < 0) { 362 + cursor += Math.abs(value); 363 + } else { 364 + if (tokenIdx < tokens.length) { 365 + const token = tokens[tokenIdx]; 366 + facets.push({ 367 + index: { 368 + byteStart: token.textSpan.byteStart, 369 + byteEnd: token.textSpan.byteEnd, 370 + }, 371 + features: [ 372 + { 373 + $type: 'tv.ionosphere.facet#timestamp', 374 + startTime: cursor * 1_000_000, // ms → ns 375 + endTime: (cursor + value) * 1_000_000, 376 + }, 377 + ], 378 + }); 379 + cursor += value; 380 + tokenIdx++; 381 + } 382 + } 383 + } 334 384 } 335 385 336 386 // 2. Sentence facets

Configure Feed

Configure Feed