Update plan: prioritize lenses for forwards-compatibility · blaine.bsky.social/ionosphere@6ef168c

+283 -24

1 changed file

expand all

docs

superpowers

plans

2026-03-30-ionosphere-implementation.md

+283 -24

docs/superpowers/plans/2026-03-30-ionosphere-implementation.md

··· 483 483 484 484 --- 485 485 486 + ### Task 4: Define lens specifications 487 + 488 + Lenses are the insulation layer between upstream source lexicons and the internal ionosphere data model. When Streamplace's lexicons change (they've indicated they will), only the lens rules need updating — the rest of the pipeline stays stable. 489 + 490 + **Files:** 491 + - Create: `formats/tv.ionosphere/lenses/schedule-to-talk.lens.json` 492 + - Create: `formats/tv.ionosphere/lenses/vod-to-talk.lens.json` 493 + - Create: `formats/tv.ionosphere/lenses/transcript-to-document.lens.json` 494 + 495 + - [ ] **Step 1: Create schedule-to-talk lens** 496 + 497 + Maps `community.lexicon.calendar.event` fields to `tv.ionosphere.talk` fields. This is where the upstream schedule format is absorbed. 498 + 499 + ```json 500 + { 501 + "$type": "org.relationaltext.lens", 502 + "id": "community.lexicon.calendar.event.to.tv.ionosphere.talk.v1", 503 + "description": "Transform ATmosphereConf schedule events into ionosphere talk records", 504 + "source": "community.lexicon.calendar.event", 505 + "target": "tv.ionosphere.talk", 506 + "invertible": false, 507 + "rules": [ 508 + { 509 + "match": { "name": "name" }, 510 + "replace": { "name": "title" } 511 + }, 512 + { 513 + "match": { "name": "description" }, 514 + "replace": { "name": "description" } 515 + }, 516 + { 517 + "match": { "name": "startsAt" }, 518 + "replace": { "name": "startsAt" } 519 + }, 520 + { 521 + "match": { "name": "endsAt" }, 522 + "replace": { "name": "endsAt" } 523 + }, 524 + { 525 + "match": { "name": "additionalData.room" }, 526 + "replace": { "name": "room" } 527 + }, 528 + { 529 + "match": { "name": "additionalData.category" }, 530 + "replace": { "name": "category" } 531 + }, 532 + { 533 + "match": { "name": "additionalData.type" }, 534 + "replace": { "name": "talkType" } 535 + }, 536 + { 537 + "match": { "name": "additionalData.speakers" }, 538 + "replace": { "name": "speakers" } 539 + } 540 + ] 541 + } 542 + ``` 543 + 544 + - [ ] **Step 2: Create vod-to-talk lens** 545 + 546 + Maps `place.stream.video` fields to the video-related fields on `tv.ionosphere.talk`. 547 + 548 + ```json 549 + { 550 + "$type": "org.relationaltext.lens", 551 + "id": "place.stream.video.to.tv.ionosphere.talk.v1", 552 + "description": "Map Streamplace VOD record fields to ionosphere talk video metadata", 553 + "source": "place.stream.video", 554 + "target": "tv.ionosphere.talk", 555 + "invertible": false, 556 + "rules": [ 557 + { 558 + "match": { "name": "title" }, 559 + "replace": { "name": "title" } 560 + }, 561 + { 562 + "match": { "name": "duration" }, 563 + "replace": { "name": "duration" } 564 + }, 565 + { 566 + "match": { "name": "creator" }, 567 + "replace": { "name": "streamCreator" } 568 + } 569 + ] 570 + } 571 + ``` 572 + 573 + - [ ] **Step 3: Create transcript-to-document lens** 574 + 575 + Maps raw transcript facets to ionosphere document facets. Initially a passthrough — the transcript word timestamps map directly to `tv.ionosphere.facet#timestamp`. 576 + 577 + ```json 578 + { 579 + "$type": "org.relationaltext.lens", 580 + "id": "transcript.to.tv.ionosphere.document.v1", 581 + "description": "Map raw transcript timing data to ionosphere document timestamp facets", 582 + "source": "transcript.raw", 583 + "target": "tv.ionosphere.facet", 584 + "passthrough": "keep", 585 + "rules": [ 586 + { 587 + "match": { "name": "word-timing" }, 588 + "replace": { "name": "timestamp" } 589 + } 590 + ] 591 + } 592 + ``` 593 + 594 + - [ ] **Step 4: Create lens loader utility** 595 + 596 + Create `formats/tv.ionosphere/ts/lenses.ts`: 597 + 598 + ```typescript 599 + import { readFileSync } from "node:fs"; 600 + import path from "node:path"; 601 + 602 + export interface LensSpec { 603 + $type: string; 604 + id: string; 605 + description: string; 606 + source: string; 607 + target: string; 608 + invertible?: boolean; 609 + passthrough?: "keep" | "drop"; 610 + rules: LensRule[]; 611 + } 612 + 613 + export interface LensRule { 614 + match: { name: string }; 615 + replace: { name: string }; 616 + } 617 + 618 + const LENS_DIR = path.resolve(import.meta.dirname, "../lenses"); 619 + 620 + export function loadLens(filename: string): LensSpec { 621 + const raw = readFileSync(path.join(LENS_DIR, filename), "utf-8"); 622 + return JSON.parse(raw); 623 + } 624 + 625 + /** 626 + * Apply a lens to transform a source record's fields to target field names. 627 + * Returns a new object with renamed keys per the lens rules. 628 + * Fields not matched by any rule are kept or dropped per `passthrough`. 629 + */ 630 + export function applyLens( 631 + lens: LensSpec, 632 + source: Record<string, any> 633 + ): Record<string, any> { 634 + const result: Record<string, any> = {}; 635 + const matched = new Set<string>(); 636 + 637 + for (const rule of lens.rules) { 638 + const sourceName = rule.match.name; 639 + // Support dotted paths (e.g., "additionalData.room") 640 + const value = getNestedValue(source, sourceName); 641 + if (value !== undefined) { 642 + result[rule.replace.name] = value; 643 + matched.add(sourceName.split(".")[0]); 644 + } 645 + } 646 + 647 + // Handle passthrough 648 + if (lens.passthrough === "keep") { 649 + for (const [key, value] of Object.entries(source)) { 650 + if (!matched.has(key) && !(key in result)) { 651 + result[key] = value; 652 + } 653 + } 654 + } 655 + 656 + return result; 657 + } 658 + 659 + function getNestedValue(obj: any, path: string): any { 660 + const parts = path.split("."); 661 + let current = obj; 662 + for (const part of parts) { 663 + if (current == null) return undefined; 664 + current = current[part]; 665 + } 666 + return current; 667 + } 668 + ``` 669 + 670 + - [ ] **Step 5: Add lens loader test** 671 + 672 + Create `formats/tv.ionosphere/ts/lenses.test.ts`: 673 + 674 + ```typescript 675 + import { describe, it, expect } from "vitest"; 676 + import { applyLens, type LensSpec } from "./lenses.js"; 677 + 678 + describe("applyLens", () => { 679 + const lens: LensSpec = { 680 + $type: "org.relationaltext.lens", 681 + id: "test", 682 + description: "test lens", 683 + source: "source", 684 + target: "target", 685 + rules: [ 686 + { match: { name: "name" }, replace: { name: "title" } }, 687 + { match: { name: "additionalData.room" }, replace: { name: "room" } }, 688 + ], 689 + }; 690 + 691 + it("renames fields per rules", () => { 692 + const result = applyLens(lens, { name: "Hello" }); 693 + expect(result.title).toBe("Hello"); 694 + expect(result.name).toBeUndefined(); 695 + }); 696 + 697 + it("handles dotted paths", () => { 698 + const result = applyLens(lens, { 699 + name: "Test", 700 + additionalData: { room: "Room 1", type: "presentation" }, 701 + }); 702 + expect(result.title).toBe("Test"); 703 + expect(result.room).toBe("Room 1"); 704 + }); 705 + 706 + it("drops unmatched fields by default", () => { 707 + const result = applyLens(lens, { name: "Test", extra: "value" }); 708 + expect(result.extra).toBeUndefined(); 709 + }); 710 + 711 + it("keeps unmatched fields with passthrough=keep", () => { 712 + const keepLens = { ...lens, passthrough: "keep" as const }; 713 + const result = applyLens(keepLens, { name: "Test", extra: "value" }); 714 + expect(result.extra).toBe("value"); 715 + }); 716 + }); 717 + ``` 718 + 719 + - [ ] **Step 6: Run tests** 720 + 721 + ```bash 722 + cd formats/tv.ionosphere && pnpm test 723 + ``` 724 + 725 + - [ ] **Step 7: Update format package exports** 726 + 727 + Add to `formats/tv.ionosphere/package.json` exports: 728 + ```json 729 + "./lenses": "./ts/lenses.ts" 730 + ``` 731 + 732 + - [ ] **Step 8: Commit** 733 + 734 + ```bash 735 + git add formats/tv.ionosphere/lenses/ formats/tv.ionosphere/ts/lenses.ts formats/tv.ionosphere/ts/lenses.test.ts formats/tv.ionosphere/package.json 736 + git commit -m "feat: lens specifications and loader for source lexicon transformation" 737 + ``` 738 + 486 739 ### Deferred from Chunk 1 487 740 488 - - **Lens files** (`schedule-to-talk.lens.json`, `transcript-to-document.lens.json`): The lens transformation logic is currently implemented procedurally in `ingest.ts` and `assemble.ts`. Declarative lens JSON specs will be formalized once the transformation rules are stable. This matches pannacotta's development pattern where lenses were extracted from working code. 489 741 - **panproto**: Schema versioning will be integrated once lexicons stabilize past their initial revision. 490 742 - **Pretext**: Transcript layout integration depends on evaluating Pretext's available API surface area, which is currently undocumented. The plan uses a custom `TranscriptView` component that can be swapped for Pretext later. 491 743 ··· 493 745 494 746 ## Chunk 2: Appview Scaffold & Data Ingest 495 747 496 - Creates the appview app, sets up SQLite schema, and implements ingestion of source data from Streamplace VODs and ATmosphereConf schedule records. 748 + Creates the appview app, sets up SQLite schema, and implements ingestion of source data from Streamplace VODs and ATmosphereConf schedule records. The ingest pipeline uses the lens loader from Chunk 1 to transform source records. 497 749 498 - ### Task 4: Scaffold the appview app 750 + ### Task 5: Scaffold the appview app 499 751 500 752 **Files:** 501 753 - Create: `apps/ionosphere-appview/package.json` ··· 797 1049 git commit -m "feat: scaffold appview with SQLite schema and REST API" 798 1050 ``` 799 1051 800 - ### Task 5: Implement data ingest from AT Protocol 1052 + ### Task 6: Implement data ingest from AT Protocol 801 1053 802 1054 **Files:** 803 1055 - Create: `apps/ionosphere-appview/src/ingest.ts` ··· 995 1247 ```typescript 996 1248 import { openDb, migrate } from "./db.js"; 997 1249 import { correlate, type ScheduleEvent, type VodRecord } from "./correlate.js"; 1250 + import { loadLens, applyLens } from "@ionosphere/format/lenses"; 1251 + 1252 + const scheduleLens = loadLens("schedule-to-talk.lens.json"); 1253 + const vodLens = loadLens("vod-to-talk.lens.json"); 998 1254 999 1255 const SCHEDULE_DID = "did:plc:3xewinw4wtimo2lqfy5fm5sw"; 1000 1256 const SCHEDULE_COLLECTION = "community.lexicon.calendar.event"; ··· 1046 1302 const type = ad?.type || ""; 1047 1303 if (["info", "food"].includes(type)) return null; 1048 1304 1305 + // Apply lens to transform source fields to internal names 1306 + const mapped = applyLens(scheduleLens, v); 1307 + 1049 1308 return { 1050 1309 uri: record.uri, 1051 - name: v.name, 1052 - startsAt: v.startsAt, 1053 - endsAt: v.endsAt, 1054 - type, 1055 - room: ad?.room || "", 1056 - category: ad?.category || "", 1057 - speakers: ad?.speakers || [], 1058 - description: v.description || "", 1310 + name: mapped.title, 1311 + startsAt: mapped.startsAt, 1312 + endsAt: mapped.endsAt, 1313 + type: mapped.talkType || "", 1314 + room: mapped.room || "", 1315 + category: mapped.category || "", 1316 + speakers: mapped.speakers || [], 1317 + description: mapped.description || "", 1059 1318 }; 1060 1319 } 1061 1320 ··· 1233 1492 1234 1493 Sets up the Next.js SSG frontend with basic page routes and a working video player. After this chunk, you can browse talks and watch videos. 1235 1494 1236 - ### Task 6: Scaffold Next.js app 1495 + ### Task 7: Scaffold Next.js app 1237 1496 1238 1497 **Files:** 1239 1498 - Create: `apps/ionosphere/package.json` ··· 1491 1750 git commit -m "feat: scaffold Next.js frontend with talk listing" 1492 1751 ``` 1493 1752 1494 - ### Task 7: Talk page with video player 1753 + ### Task 8: Talk page with video player 1495 1754 1496 1755 **Files:** 1497 1756 - Create: `apps/ionosphere/src/app/talks/page.tsx` ··· 1722 1981 git commit -m "feat: talk pages with HLS video player" 1723 1982 ``` 1724 1983 1725 - ### Task 8: Speaker and concept pages 1984 + ### Task 9: Speaker and concept pages 1726 1985 1727 1986 **Files:** 1728 1987 - Create: `apps/ionosphere/src/app/speakers/page.tsx` ··· 1920 2179 1921 2180 Implements audio extraction from HLS streams and transcription with word-level timestamps. After this chunk, talks have transcripts stored in the database. 1922 2181 1923 - ### Task 9: Audio extraction from HLS 2182 + ### Task 10: Audio extraction from HLS 1924 2183 1925 2184 **Files:** 1926 2185 - Create: `apps/ionosphere-appview/src/extract-audio.ts` ··· 2001 2260 git commit -m "feat: audio extraction from HLS streams via ffmpeg" 2002 2261 ``` 2003 2262 2004 - ### Task 10: Transcription integration 2263 + ### Task 11: Transcription integration 2005 2264 2006 2265 **Files:** 2007 2266 - Create: `apps/ionosphere-appview/src/transcribe.ts` ··· 2130 2389 2131 2390 Converts raw transcripts into RelationalText documents with timestamp facets, and implements the frontend timestamp sync. 2132 2391 2133 - ### Task 11: Document assembly — transcript to RelationalText 2392 + ### Task 12: Document assembly — transcript to RelationalText 2134 2393 2135 2394 **Files:** 2136 2395 - Create: `formats/tv.ionosphere/ts/assemble.ts` ··· 2270 2529 git commit -m "feat: assemble RelationalText documents from transcripts with timestamp facets" 2271 2530 ``` 2272 2531 2273 - ### Task 12: Timestamp provider and transcript sync in frontend 2532 + ### Task 13: Timestamp provider and transcript sync in frontend 2274 2533 2275 2534 **Files:** 2276 2535 - Create: `apps/ionosphere/src/app/components/TimestampProvider.tsx` ··· 2489 2748 2490 2749 | Chunk | Tasks | What it delivers | 2491 2750 |-------|-------|------------------| 2492 - | 1: Scaffold & Lexicons | 1-3 | Workspace, lexicons, format-lexicon | 2493 - | 2: Appview & Ingest | 4-5 | SQLite schema, REST API, data ingest pipeline | 2494 - | 3: Frontend | 6-8 | Next.js SSG, talk/speaker/concept pages, video player | 2495 - | 4: Transcription | 9-10 | Audio extraction, transcription pipeline skeleton | 2496 - | 5: Document Assembly | 11-12 | RelationalText documents, timestamp sync, transcript view | 2751 + | 1: Scaffold & Lexicons | 1-4 | Workspace, lexicons, format-lexicon, lens specs + loader | 2752 + | 2: Appview & Ingest | 5-6 | SQLite schema, REST API, lens-driven data ingest pipeline | 2753 + | 3: Frontend | 7-9 | Next.js SSG, talk/speaker/concept pages, video player | 2754 + | 4: Transcription | 10-11 | Audio extraction, transcription pipeline skeleton | 2755 + | 5: Document Assembly | 12-13 | RelationalText documents, timestamp sync, transcript view | 2497 2756 | 6: Enrichment | (future) | LLM annotation, concept extraction, knowledge graph |

Configure Feed

Configure Feed