feat: chunk segmentation records for PDS body limit compliance

+26 -23

apps/ionosphere-appview/src/__tests__/layers-pub.test.ts

··· 27 27 expect(expression.createdAt).toBeDefined(); 28 28 }); 29 29 30 - it('produces a segmentation record with textSpan-only tokens', async () => { 31 - const { segmentation } = await transcriptToLayersPub(transcript, did, talkRkey); 32 - expect(segmentation.$type).toBe('pub.layers.segmentation.segmentation'); 33 - expect(segmentation.expression).toBe( 30 + it('produces segmentation records with textSpan-only tokens', async () => { 31 + const { segmentations } = await transcriptToLayersPub(transcript, did, talkRkey); 32 + expect(segmentations).toHaveLength(1); // small transcript = 1 chunk 33 + 34 + const seg = segmentations[0]; 35 + expect(seg.$type).toBe('pub.layers.segmentation.segmentation'); 36 + expect(seg.expression).toBe( 34 37 'at://did:plc:test/pub.layers.expression.expression/test-talk-expression' 35 38 ); 36 - expect(segmentation.tokenizations).toHaveLength(1); 37 39 38 - const tok = segmentation.tokenizations[0]; 40 + const tok = seg.tokenizations[0]; 39 41 expect(tok.kind).toBe('word'); 40 42 expect(tok.tokens).toHaveLength(4); 41 43 42 - // Check first token — textSpan only, no temporalSpan 44 + // Check first token — textSpan only (word text recoverable via expression.text + byte offsets) 43 45 expect(tok.tokens[0].tokenIndex).toBe(0); 44 - expect(tok.tokens[0].text).toBe('Hello'); 45 46 expect(tok.tokens[0].textSpan.byteStart).toBe(0); 46 47 expect(tok.tokens[0].textSpan.byteEnd).toBe(5); 47 48 48 49 // Check third token (after gap) — byte offsets 49 - expect(tok.tokens[2].text).toBe('foo'); 50 50 expect(tok.tokens[2].textSpan.byteStart).toBe(12); // "Hello world " = 12 bytes 51 51 expect(tok.tokens[2].textSpan.byteEnd).toBe(15); 52 52 }); 53 53 54 - it('produces a separate temporal segmentation record', async () => { 55 - const { temporal } = await transcriptToLayersPub(transcript, did, talkRkey); 56 - expect(temporal.$type).toBe('pub.layers.segmentation.segmentation'); 57 - expect(temporal.tokenizations).toHaveLength(1); 54 + it('produces separate temporal segmentation records', async () => { 55 + const { temporals } = await transcriptToLayersPub(transcript, did, talkRkey); 56 + expect(temporals).toHaveLength(1); // small transcript = 1 chunk 57 + 58 + const temp = temporals[0]; 59 + expect(temp.$type).toBe('pub.layers.segmentation.segmentation'); 58 60 59 - const tok = temporal.tokenizations[0]; 61 + const tok = temp.tokenizations[0]; 60 62 expect(tok.kind).toBe('word-temporal'); 61 63 expect(tok.tokens).toHaveLength(4); 62 64 ··· 171 173 talkUri: `at://${did}/tv.ionosphere.talk/${rkey}`, 172 174 }; 173 175 174 - const { expression, segmentation, temporal } = await transcriptToLayersPub(transcriptRecord, did, rkey); 176 + const { expression, segmentations, temporals } = await transcriptToLayersPub(transcriptRecord, did, rkey); 175 177 const expressionUri = `at://${did}/pub.layers.expression.expression/${rkey}-expression`; 176 178 const layers = await nlpToAnnotationLayers(nlpData, did, rkey, expressionUri); 177 179 178 - // Verify all 7 records have correct $type 180 + // Verify records have correct $type 179 181 expect(expression.$type).toBe('pub.layers.expression.expression'); 180 - expect(segmentation.$type).toBe('pub.layers.segmentation.segmentation'); 181 - expect(temporal.$type).toBe('pub.layers.segmentation.segmentation'); 182 + for (const seg of segmentations) expect(seg.$type).toBe('pub.layers.segmentation.segmentation'); 183 + for (const temp of temporals) expect(temp.$type).toBe('pub.layers.segmentation.segmentation'); 182 184 expect(layers.sentences.$type).toBe('pub.layers.annotation.annotationLayer'); 183 185 expect(layers.paragraphs.$type).toBe('pub.layers.annotation.annotationLayer'); 184 186 expect(layers.entities.$type).toBe('pub.layers.annotation.annotationLayer'); 185 187 expect(layers.topics.$type).toBe('pub.layers.annotation.annotationLayer'); 186 188 187 - // Verify real data produces non-trivial results 188 - expect(segmentation.tokenizations[0].tokens.length).toBeGreaterThan(100); 189 - expect(temporal.tokenizations[0].tokens.length).toBeGreaterThan(100); 189 + // Verify real data produces non-trivial results (keynote has 8040 tokens = 2 chunks) 190 + const totalTokens = segmentations.reduce((n, s) => n + s.tokenizations[0].tokens.length, 0); 191 + expect(totalTokens).toBeGreaterThan(100); 192 + expect(segmentations.length).toBe(temporals.length); 190 193 expect(layers.sentences.annotations.length).toBeGreaterThan(10); 191 194 expect(layers.entities.annotations.length).toBeGreaterThan(10); 192 195 }); ··· 229 232 timings: compact.timings, 230 233 talkUri: `at://${did}/tv.ionosphere.talk/${rkey}`, 231 234 }; 232 - const { expression, segmentation } = await transcriptToLayersPub(transcriptRecord, did, rkey); 235 + const { expression, segmentations } = await transcriptToLayersPub(transcriptRecord, did, rkey); 233 236 const expressionUri = `at://${did}/pub.layers.expression.expression/${rkey}-expression`; 234 237 const layers = await nlpToAnnotationLayers(nlpData, did, rkey, expressionUri); 235 - const lensDoc = await layersPubToDocument(expression, segmentation, layers, compact); 238 + const lensDoc = await layersPubToDocument(expression, segmentations, layers, compact); 236 239 237 240 // Compare text 238 241 expect(lensDoc.text).toBe(directDoc.text);

+9 -9

apps/ionosphere-appview/src/layers-indexer.ts

··· 113 113 .get(expressionUri) as any; 114 114 if (!expr) return; 115 115 116 - // 2. Look up segmentation 117 - const seg = db 118 - .prepare("SELECT * FROM layers_segmentations WHERE expression_uri = ?") 119 - .get(expressionUri) as any; 120 - if (!seg) return; 116 + // 2. Look up segmentation chunks (may be multiple for large transcripts) 117 + const segRows = db 118 + .prepare("SELECT * FROM layers_segmentations WHERE expression_uri = ? AND rkey NOT LIKE '%-temporal%' ORDER BY rkey") 119 + .all(expressionUri) as any[]; 120 + if (segRows.length === 0) return; 121 121 122 122 // 3. Look up all annotation layers 123 123 const annRows = db ··· 164 164 createdAt: expr.created_at, 165 165 }; 166 166 167 - const segmentationRecord: SegmentationRecord = { 168 - $type: "pub.layers.segmentation.segmentation", 167 + const segmentationRecords: SegmentationRecord[] = segRows.map((seg: any) => ({ 168 + $type: "pub.layers.segmentation.segmentation" as const, 169 169 expression: expressionUri, 170 170 tokenizations: JSON.parse(seg.tokens_json), 171 171 createdAt: seg.created_at, 172 - }; 172 + })); 173 173 174 174 // Fill missing layers with empty annotations so layersPubToDocument gets 175 175 // the full AnnotationLayersResult shape it expects ··· 206 206 207 207 const document = await layersPubToDocument( 208 208 expressionRecord, 209 - segmentationRecord, 209 + segmentationRecords, 210 210 fullLayers, 211 211 compact, 212 212 );

+16 -4

apps/ionosphere-appview/src/publish.ts

··· 214 214 talkUri: `at://${did}/tv.ionosphere.talk/${talk.rkey}`, 215 215 }; 216 216 217 - const { expression, segmentation, temporal } = await transcriptToLayersPub(transcriptRecord, did, talk.rkey); 217 + const { expression, segmentations, temporals } = await transcriptToLayersPub(transcriptRecord, did, talk.rkey); 218 218 const expressionUri = `at://${did}/pub.layers.expression.expression/${talk.rkey}-expression`; 219 219 const layers = await nlpToAnnotationLayers(nlpData, did, talk.rkey, expressionUri); 220 220 221 + const segPuts = segmentations.map((seg, i) => 222 + pds.putRecord("pub.layers.segmentation.segmentation", 223 + segmentations.length === 1 ? `${talk.rkey}-segmentation` : `${talk.rkey}-segmentation-${i}`, 224 + seg) 225 + ); 226 + const temporalPuts = temporals.map((temp, i) => 227 + pds.putRecord("pub.layers.segmentation.segmentation", 228 + temporals.length === 1 ? `${talk.rkey}-temporal` : `${talk.rkey}-temporal-${i}`, 229 + temp) 230 + ); 231 + 221 232 await Promise.all([ 222 233 pds.putRecord("pub.layers.expression.expression", `${talk.rkey}-expression`, expression), 223 - pds.putRecord("pub.layers.segmentation.segmentation", `${talk.rkey}-segmentation`, segmentation), 224 - pds.putRecord("pub.layers.segmentation.segmentation", `${talk.rkey}-temporal`, temporal), 234 + ...segPuts, 235 + ...temporalPuts, 225 236 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-sentences`, layers.sentences), 226 237 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-paragraphs`, layers.paragraphs), 227 238 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-entities`, layers.entities), 228 239 pds.putRecord("pub.layers.annotation.annotationLayer", `${talk.rkey}-topics`, layers.topics), 229 240 ]); 230 241 231 - console.log(` layers.pub: ${talk.rkey} (7 records)`); 242 + const totalRecords = 1 + segmentations.length + temporals.length + 4; 243 + console.log(` layers.pub: ${talk.rkey} (${totalRecords} records${segmentations.length > 1 ? `, ${segmentations.length} seg chunks` : ''})`); 232 244 layersCount++; 233 245 } 234 246 console.log(`Published layers.pub records for ${layersCount} talks.`);

+34 -21

formats/tv.ionosphere/ts/layers-pub.ts

··· 52 52 53 53 export interface TextToken { 54 54 tokenIndex: number; 55 - text: string; 56 55 textSpan: TokenSpan; 57 56 } 58 57 ··· 85 84 createdAt: string; 86 85 } 87 86 87 + /** Maximum tokens per segmentation record to stay under PDS body limits (~200KB CBOR) */ 88 + const MAX_TOKENS_PER_RECORD = 2000; 89 + 88 90 export interface LayersPubResult { 89 91 expression: ExpressionRecord; 90 - segmentation: SegmentationRecord; 91 - temporal: TemporalSegmentationRecord; 92 + segmentations: SegmentationRecord[]; 93 + temporals: TemporalSegmentationRecord[]; 92 94 } 93 95 94 96 /** ··· 153 155 154 156 textTokens.push({ 155 157 tokenIndex: wordIndex, 156 - text: word, 157 158 textSpan: { byteStart, byteEnd }, 158 159 }); 159 160 ··· 172 173 173 174 const expressionUri = `at://${did}/pub.layers.expression.expression/${talkRkey}-expression`; 174 175 175 - const segmentation: SegmentationRecord = { 176 - $type: 'pub.layers.segmentation.segmentation', 177 - expression: expressionUri, 178 - tokenizations: [{ kind: 'word', tokens: textTokens }], 179 - createdAt: now, 180 - }; 176 + // Chunk tokens to stay under PDS body limits 177 + const segmentations: SegmentationRecord[] = []; 178 + const temporals: TemporalSegmentationRecord[] = []; 179 + 180 + for (let i = 0; i < textTokens.length; i += MAX_TOKENS_PER_RECORD) { 181 + const textChunk = textTokens.slice(i, i + MAX_TOKENS_PER_RECORD); 182 + const temporalChunk = temporalTokens.slice(i, i + MAX_TOKENS_PER_RECORD); 183 + 184 + segmentations.push({ 185 + $type: 'pub.layers.segmentation.segmentation', 186 + expression: expressionUri, 187 + tokenizations: [{ kind: 'word', tokens: textChunk }], 188 + createdAt: now, 189 + }); 181 190 182 - const temporal: TemporalSegmentationRecord = { 183 - $type: 'pub.layers.segmentation.segmentation', 184 - expression: expressionUri, 185 - tokenizations: [{ kind: 'word-temporal', tokens: temporalTokens }], 186 - createdAt: now, 187 - }; 191 + temporals.push({ 192 + $type: 'pub.layers.segmentation.segmentation', 193 + expression: expressionUri, 194 + tokenizations: [{ kind: 'word-temporal', tokens: temporalChunk }], 195 + createdAt: now, 196 + }); 197 + } 188 198 189 - return { expression, segmentation, temporal }; 199 + return { expression, segmentations, temporals }; 190 200 } 191 201 192 202 /** ··· 343 353 */ 344 354 export async function layersPubToDocument( 345 355 expression: ExpressionRecord, 346 - segmentation: SegmentationRecord, 356 + segmentations: SegmentationRecord | SegmentationRecord[], 347 357 annotationLayers: AnnotationLayersResult, 348 358 compact?: CompactTranscript, 349 359 ): Promise<Document> { 350 360 const facets: DocumentFacet[] = []; 361 + 362 + // Merge all segmentation chunks into a single token list 363 + const segArray = Array.isArray(segmentations) ? segmentations : [segmentations]; 364 + const allTokens = segArray.flatMap((s) => s.tokenizations[0].tokens); 351 365 352 366 // 1. Timestamp facets from compact transcript timings 353 367 // Uses the same replay algorithm as decodeToDocument() — the segmentation 354 368 // provides byte offsets, the compact transcript provides timing. 355 369 if (compact) { 356 - const tokens = segmentation.tokenizations[0].tokens; 357 370 let cursor = compact.startMs; 358 371 let tokenIdx = 0; 359 372 ··· 361 374 if (value < 0) { 362 375 cursor += Math.abs(value); 363 376 } else { 364 - if (tokenIdx < tokens.length) { 365 - const token = tokens[tokenIdx]; 377 + if (tokenIdx < allTokens.length) { 378 + const token = allTokens[tokenIdx]; 366 379 facets.push({ 367 380 index: { 368 381 byteStart: token.textSpan.byteStart,

Configure Feed

Configure Feed