Add embed_json column + PostEmbed types to post_snapshots

+4

app/lib/atproto/index.ts

··· 13 13 AccountEvent, 14 14 IdentityEvent, 15 15 PostSnapshot, 16 + PostEmbed, 17 + ImagesEmbed, 18 + VideoEmbed, 19 + ExternalEmbed, 16 20 } from './types.js' 17 21 18 22 // Client wrapper

+2

app/lib/atproto/parsers/get_author_feed.ts

··· 95 95 snapshotReposts, 96 96 snapshotQuotes, 97 97 snapshotTakenAt, 98 + // Embed parsing is wired up in a later task; default to null for now. 99 + embed: null, 98 100 }) 99 101 }) 100 102

+38

app/lib/atproto/types.ts

··· 154 154 snapshotQuotes: number 155 155 /** Set to now() at parse time — the per-post watermark (spec §5) */ 156 156 snapshotTakenAt: Date 157 + /** Optional embed parsed from the post's record (images/video/external). Null = no embed. */ 158 + embed: PostEmbed | null 159 + } 160 + 161 + // --------------------------------------------------------------------------- 162 + // Post embeds (tagged union) 163 + // --------------------------------------------------------------------------- 164 + 165 + /** 166 + * Discriminated union of supported post embed types. Quote posts 167 + * (app.bsky.embed.record and recordWithMedia) are intentionally excluded — 168 + * the ingest layer drops them before a snapshot is ever created. 169 + */ 170 + export type PostEmbed = ImagesEmbed | VideoEmbed | ExternalEmbed 171 + 172 + export interface ImagesEmbed { 173 + type: 'images' 174 + items: Array<{ 175 + thumb: string 176 + fullsize: string 177 + alt: string 178 + aspectRatio?: { width: number; height: number } 179 + }> 180 + } 181 + 182 + export interface VideoEmbed { 183 + type: 'video' 184 + thumbnail: string 185 + alt: string 186 + aspectRatio?: { width: number; height: number } 187 + } 188 + 189 + export interface ExternalEmbed { 190 + type: 'external' 191 + uri: string 192 + title: string 193 + description: string 194 + thumb: string | null 157 195 }

+1 -1

app/lib/clickhouse/index.ts

··· 6 6 EngagementEventRow, 7 7 ClickHouseConfig, 8 8 } from './types.js' 9 - export type { PostSnapshot } from '#lib/atproto/index' 9 + export type { PostSnapshot, PostEmbed } from '#lib/atproto/index'

+10 -3

app/lib/clickhouse/store.ts

··· 1 1 import { createClient } from '@clickhouse/client' 2 2 import type { ClickHouseClient } from '@clickhouse/client' 3 - import type { PostSnapshot } from '#lib/atproto/index' 3 + import type { PostEmbed, PostSnapshot } from '#lib/atproto/index' 4 4 import type { 5 5 ClickHouseConfig, 6 6 EngagementEventRow, ··· 19 19 post_created_at: string // DateTime64 comes back as a string in JSONEachRow 20 20 likes: string // aggregates come back as strings in ClickHouse JSON 21 21 reposts: string 22 + embed_json: string // empty string = no embed 22 23 } 23 24 24 25 // --------------------------------------------------------------------------- ··· 34 35 s.post_uri, 35 36 s.post_text, 36 37 s.post_created_at, 38 + s.embed_json, 37 39 s.snapshot_likes 38 40 + countIf(e.kind = 'like' AND e.event_created_at > s.snapshot_taken_at) 39 41 AS likes, ··· 52 54 AND s.is_deleted = 0` 53 55 54 56 const GROUP_BY = ` 55 - GROUP BY s.post_uri, s.post_text, s.post_created_at, 57 + GROUP BY s.post_uri, s.post_text, s.post_created_at, s.embed_json, 56 58 s.snapshot_likes, s.snapshot_reposts, s.snapshot_taken_at` 57 59 58 60 const LIMIT = ` ··· 185 187 postCreatedAt: new Date(row.post_created_at), 186 188 likes: Number(row.likes), 187 189 reposts: Number(row.reposts), 190 + // Empty string is the canonical "no embed" sentinel — don't JSON.parse(''). 191 + embed: row.embed_json ? (JSON.parse(row.embed_json) as PostEmbed) : null, 188 192 })) 189 193 } 190 194 ··· 210 214 snapshot_quotes: s.snapshotQuotes, 211 215 snapshot_taken_at: dateToClickHouseStr(s.snapshotTakenAt), 212 216 is_deleted: 0, 217 + embed_json: s.embed ? JSON.stringify(s.embed) : '', 213 218 })) 214 219 215 220 try { ··· 288 293 snapshot_quotes: 0, 289 294 snapshot_taken_at: now, 290 295 is_deleted: 1, 296 + embed_json: '', 291 297 }, 292 298 ], 293 299 format: 'JSONEachRow', ··· 329 335 snapshot_reposts, 330 336 snapshot_quotes, 331 337 now64(6), 332 - toUInt8(1) 338 + toUInt8(1), 339 + '' 333 340 FROM post_snapshots FINAL 334 341 WHERE post_author_did = {authorDid:String} 335 342 AND is_deleted = 0

+3

app/lib/clickhouse/types.ts

··· 1 1 /** 2 2 * Public types for the ClickHouse package. 3 3 */ 4 + import type { PostEmbed } from '#lib/atproto/index' 4 5 5 6 // --------------------------------------------------------------------------- 6 7 // Query types ··· 21 22 postCreatedAt: Date 22 23 likes: number 23 24 reposts: number 25 + /** Parsed embed (images/video/external) or null if the post had no embed. */ 26 + embed: PostEmbed | null 24 27 } 25 28 26 29 // ---------------------------------------------------------------------------

+2

app/services/jetstream_consumer.ts

··· 376 376 snapshotReposts: 0, 377 377 snapshotQuotes: 0, 378 378 snapshotTakenAt: this.deps.now(), 379 + // Embed parsing is wired up in a later task; default to null for now. 380 + embed: null, 379 381 }) 380 382 381 383 this.advancePendingCursor(timeUs)

+1

database/clickhouse/003_add_embed_to_post_snapshots.sql

··· 1 + ALTER TABLE post_snapshots ADD COLUMN IF NOT EXISTS embed_json String DEFAULT ''

+4

tests/functional/profile_controller.spec.ts

··· 302 302 snapshotReposts: 7, 303 303 snapshotQuotes: 0, 304 304 snapshotTakenAt: new Date('2024-01-15T12:00:00Z'), 305 + embed: null, 305 306 }, 306 307 ]) 307 308 ··· 340 341 snapshotReposts: 2, 341 342 snapshotQuotes: 0, 342 343 snapshotTakenAt: new Date(), 344 + embed: null, 343 345 }, 344 346 { 345 347 postUri: 'at://did:plc:test002/app.bsky.feed.post/old', ··· 350 352 snapshotReposts: 9999, 351 353 snapshotQuotes: 0, 352 354 snapshotTakenAt: new Date('2020-01-01T00:00:00Z'), 355 + embed: null, 353 356 }, 354 357 ]) 355 358 ··· 424 427 snapshotReposts: 5, 425 428 snapshotQuotes: 0, 426 429 snapshotTakenAt: new Date(), 430 + embed: null, 427 431 }, 428 432 ]) 429 433

+127

tests/unit/clickhouse_store.spec.ts

··· 13 13 import { fileURLToPath } from 'node:url' 14 14 import { join } from 'node:path' 15 15 import { ClickHouseStore } from '#lib/clickhouse/index' 16 + import type { ExternalEmbed, ImagesEmbed, VideoEmbed } from '#lib/atproto/index' 16 17 import { aLikeEvent, aSnapshot } from './clickhouse_store_fixtures.js' 17 18 18 19 // --------------------------------------------------------------------------- ··· 587 588 assert.instanceOf(thrown, Error) 588 589 const error = thrown as Error & { cause?: unknown } 589 590 assert.exists(error.cause, 'Error.cause should be set') 591 + }).skip(async () => !(await isClickHouseAvailable()), 'ClickHouse not available') 592 + 593 + // ------------------------------------------------------------------------- 594 + // Embed round-trip tests 595 + // ------------------------------------------------------------------------- 596 + 597 + test('getTopPosts round-trips an ImagesEmbed', async ({ assert }) => { 598 + const author = 'did:plc:author_embed_images' 599 + const embed: ImagesEmbed = { 600 + type: 'images', 601 + items: [ 602 + { 603 + thumb: 'https://cdn.bsky.app/img/feed_thumbnail/plain/did:plc:x/cid1', 604 + fullsize: 'https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:x/cid1', 605 + alt: 'first image alt', 606 + aspectRatio: { width: 1200, height: 800 }, 607 + }, 608 + { 609 + thumb: 'https://cdn.bsky.app/img/feed_thumbnail/plain/did:plc:x/cid2', 610 + fullsize: 'https://cdn.bsky.app/img/feed_fullsize/plain/did:plc:x/cid2', 611 + alt: 'second image alt', 612 + aspectRatio: { width: 640, height: 480 }, 613 + }, 614 + ], 615 + } 616 + 617 + await store.insertPostSnapshots([ 618 + aSnapshot({ 619 + postAuthorDid: author, 620 + postUri: `at://${author}/app.bsky.feed.post/images1`, 621 + snapshotLikes: 1, 622 + embed, 623 + }), 624 + ]) 625 + 626 + const results = await store.getTopPosts({ authorDid: author, kind: 'likes' }) 627 + assert.equal(results.length, 1) 628 + assert.deepEqual(results[0].embed, embed) 629 + }).skip(async () => !(await isClickHouseAvailable()), 'ClickHouse not available') 630 + 631 + test('getTopPosts round-trips a VideoEmbed', async ({ assert }) => { 632 + const author = 'did:plc:author_embed_video' 633 + const embed: VideoEmbed = { 634 + type: 'video', 635 + thumbnail: 'https://video.bsky.app/watch/did%3Aplc%3Aabc/bafyvideo/thumbnail.jpg', 636 + alt: 'a short clip', 637 + aspectRatio: { width: 1920, height: 1080 }, 638 + } 639 + 640 + await store.insertPostSnapshots([ 641 + aSnapshot({ 642 + postAuthorDid: author, 643 + postUri: `at://${author}/app.bsky.feed.post/video1`, 644 + snapshotLikes: 1, 645 + embed, 646 + }), 647 + ]) 648 + 649 + const results = await store.getTopPosts({ authorDid: author, kind: 'likes' }) 650 + assert.equal(results.length, 1) 651 + assert.deepEqual(results[0].embed, embed) 652 + }).skip(async () => !(await isClickHouseAvailable()), 'ClickHouse not available') 653 + 654 + test('getTopPosts round-trips an ExternalEmbed with thumb', async ({ assert }) => { 655 + const author = 'did:plc:author_embed_external' 656 + const embed: ExternalEmbed = { 657 + type: 'external', 658 + uri: 'https://example.com/article', 659 + title: 'Example Article', 660 + description: 'A short description of the linked article.', 661 + thumb: 'https://cdn.bsky.app/img/feed_thumbnail/plain/did:plc:x/cidthumb', 662 + } 663 + 664 + await store.insertPostSnapshots([ 665 + aSnapshot({ 666 + postAuthorDid: author, 667 + postUri: `at://${author}/app.bsky.feed.post/ext1`, 668 + snapshotLikes: 1, 669 + embed, 670 + }), 671 + ]) 672 + 673 + const results = await store.getTopPosts({ authorDid: author, kind: 'likes' }) 674 + assert.equal(results.length, 1) 675 + assert.deepEqual(results[0].embed, embed) 676 + }).skip(async () => !(await isClickHouseAvailable()), 'ClickHouse not available') 677 + 678 + test('getTopPosts round-trips an ExternalEmbed without thumb', async ({ assert }) => { 679 + const author = 'did:plc:author_embed_external_nothumb' 680 + const embed: ExternalEmbed = { 681 + type: 'external', 682 + uri: 'https://example.com/no-thumb', 683 + title: 'No Thumb Article', 684 + description: 'Nothing to preview here.', 685 + thumb: null, 686 + } 687 + 688 + await store.insertPostSnapshots([ 689 + aSnapshot({ 690 + postAuthorDid: author, 691 + postUri: `at://${author}/app.bsky.feed.post/ext2`, 692 + snapshotLikes: 1, 693 + embed, 694 + }), 695 + ]) 696 + 697 + const results = await store.getTopPosts({ authorDid: author, kind: 'likes' }) 698 + assert.equal(results.length, 1) 699 + assert.deepEqual(results[0].embed, embed) 700 + }).skip(async () => !(await isClickHouseAvailable()), 'ClickHouse not available') 701 + 702 + test('getTopPosts returns null embed when snapshot has no embed', async ({ assert }) => { 703 + const author = 'did:plc:author_embed_null' 704 + 705 + await store.insertPostSnapshots([ 706 + aSnapshot({ 707 + postAuthorDid: author, 708 + postUri: `at://${author}/app.bsky.feed.post/noembed`, 709 + snapshotLikes: 1, 710 + embed: null, 711 + }), 712 + ]) 713 + 714 + const results = await store.getTopPosts({ authorDid: author, kind: 'likes' }) 715 + assert.equal(results.length, 1) 716 + assert.isNull(results[0].embed) 590 717 }).skip(async () => !(await isClickHouseAvailable()), 'ClickHouse not available') 591 718 592 719 // -------------------------------------------------------------------------

+3 -1

tests/unit/clickhouse_store_fixtures.ts

··· 2 2 * Test fixture builders for ClickHouseStore tests. 3 3 * All defaults are minimal valid values; override what your test cares about. 4 4 */ 5 - import type { EngagementEventRow, PostSnapshot } from '#lib/clickhouse/index' 5 + import type { EngagementEventRow, PostEmbed, PostSnapshot } from '#lib/clickhouse/index' 6 6 7 7 // --------------------------------------------------------------------------- 8 8 // PostSnapshot builder ··· 17 17 snapshotReposts?: number 18 18 snapshotQuotes?: number 19 19 snapshotTakenAt?: Date 20 + embed?: PostEmbed | null 20 21 } 21 22 22 23 let snapshotSeq = 0 ··· 34 35 snapshotReposts: overrides.snapshotReposts ?? 0, 35 36 snapshotQuotes: overrides.snapshotQuotes ?? 0, 36 37 snapshotTakenAt: overrides.snapshotTakenAt ?? new Date('2025-01-15T13:00:00Z'), 38 + embed: overrides.embed ?? null, 37 39 } 38 40 } 39 41

Configure Feed

Configure Feed