eight elle ass (#35) · sprk.so/server@d0207e9

+1

services/appview/.gitignore

··· 41 41 package-lock.json 42 42 **/*.bun 43 43 /devdb 44 + mongo-keyfile

+6 -14

services/appview/api/so/sprk/feed/getAuthorFeed.ts

··· 1 1 import { Server } from "../../../../lexicon/index.ts"; 2 2 import { AppContext } from "../../../../main.ts"; 3 - import { transformPostToPostView } from "../../../../utils/post-transformer.ts"; 3 + import { transformPostsToPostViews } from "../../../../utils/post-transformer.ts"; 4 4 import { decodeBase64, encodeBase64 } from "jsr:@std/encoding"; 5 5 6 6 export default function (server: Server, ctx: AppContext) { ··· 133 133 } 134 134 135 135 // Transform posts to feed view posts 136 - const feedViewPosts = await Promise.all( 137 - [...pinnedPosts, ...posts].map(async (post) => { 138 - const postView = await transformPostToPostView( 139 - post, 140 - ctx.db, 141 - userDid, 142 - ); 143 - 144 - return { 145 - post: postView, 146 - }; 147 - }), 136 + const feedViewPosts = await transformPostsToPostViews( 137 + [...pinnedPosts, ...posts], 138 + ctx.db, 139 + userDid, 148 140 ); 149 141 150 142 // Generate next cursor if there are more results ··· 160 152 encoding: "application/json", 161 153 body: { 162 154 cursor: nextCursor, 163 - feed: feedViewPosts, 155 + feed: feedViewPosts.map((post) => ({ post })), 164 156 }, 165 157 }; 166 158 } catch (error) {

+25 -4

services/appview/compose.dev.yaml

··· 6 6 MONGO_INITDB_ROOT_PASSWORD: mongo 7 7 MONGO_INITDB_DATABASE: dev 8 8 ports: 9 - - "27017:27017" 9 + - '27017:27017' 10 10 volumes: 11 11 - ./devdb:/data/db 12 + # Generate with `openssl rand -base64 756 > mongo-keyfile && chmod 400 mongo-keyfile && sudo chown 999:999 mongo-keyfile` 13 + - ./mongo-keyfile:/etc/mongo-keyfile:ro 14 + command: ['--replSet', 'rs0', '--keyFile', '/etc/mongo-keyfile', '--auth'] 12 15 healthcheck: 13 - test: ["CMD", "mongosh", "--eval", "'db.adminCommand(\"ping\")'"] 16 + test: ['CMD', 'mongosh', '--eval', '''db.adminCommand("ping")'''] 14 17 interval: 10s 15 18 timeout: 5s 16 19 retries: 3 17 20 restart: unless-stopped 18 21 22 + mongo-init-replica: 23 + image: mongo:8 24 + depends_on: 25 + db: 26 + condition: service_healthy 27 + entrypoint: > 28 + bash -c " 29 + echo 'initiating replica set...'; 30 + mongosh --host db:27017 -u mongo -p mongo --eval ' 31 + rs.initiate({ 32 + _id: \"rs0\", 33 + members: [{ _id: 0, host: \"db:27017\" }] 34 + }) 35 + '; 36 + echo 'replica set ready'; 37 + " 38 + restart: 'no' 39 + 19 40 app: 20 41 build: 21 42 context: . ··· 29 50 DB_USER: mongo 30 51 DB_PASSWORD: mongo 31 52 DB_NAME: dev 32 - ADMIN_PASSWORD: "00000000000000000000000000000000" 53 + ADMIN_PASSWORD: '00000000000000000000000000000000' 33 54 env_file: 34 55 - .env 35 56 ports: 36 - - "4000:3000" 57 + - '4000:3000' 37 58 depends_on: 38 59 db: 39 60 condition: service_healthy

+17

services/appview/data-plane/server/index.ts

··· 639 639 updatedAt: { type: Date, default: Date.now }, 640 640 }); 641 641 642 + export interface VideoMappingDocument extends Document { 643 + key: string; // did-cid 644 + bunnyGuid: string; 645 + postMongoId: string; 646 + } 647 + 648 + export const videoMappingSchema = new Schema<VideoMappingDocument>({ 649 + key: { type: String, required: true, unique: true, index: true }, 650 + bunnyGuid: { type: String, required: true, index: true }, 651 + postMongoId: { type: String, required: true, index: true }, 652 + }); 653 + 642 654 export interface DatabaseModels { 643 655 Like: Model<LikeDocument>; 644 656 Post: Model<PostDocument>; ··· 657 669 Actor: Model<ActorDocument>; 658 670 UserPreference: Model<UserPreferenceDocument>; 659 671 CursorState: Model<CursorStateDocument>; 672 + VideoMapping: Model<VideoMappingDocument>; 660 673 } 661 674 662 675 export class Database implements DataPlaneClient { ··· 700 713 Profile: this.connection.model<ProfileDocument>( 701 714 "Profile", 702 715 profileSchema, 716 + ), 717 + VideoMapping: this.connection.model<VideoMappingDocument>( 718 + "VideoMapping", 719 + videoMappingSchema, 703 720 ), 704 721 Audio: this.connection.model<AudioDocument>("Audio", audioSchema), 705 722 Repost: this.connection.model<RepostDocument>("Repost", repostSchema),

+26 -9

services/appview/utils/embed-transformer.ts

··· 1 1 import type * as SoSprkEmbedImages from "../lexicon/types/so/sprk/embed/images.ts"; 2 - import { EmbedImage, PostEmbed } from "../data-plane/server/index.ts"; 2 + import { 3 + EmbedImage, 4 + PostEmbed, 5 + VideoMappingDocument, 6 + } from "../data-plane/server/index.ts"; 7 + import { env } from "./env.ts"; 3 8 4 9 interface ImageTransformOptions { 5 10 /** If true, only return the first image (useful for stories) */ ··· 39 44 export function transformVideoEmbed( 40 45 embed: PostEmbed, 41 46 authorDid: string, 42 - cid: string, 47 + videoMapping?: VideoMappingDocument | null, 43 48 ) { 44 49 if (!embed.video) { 45 50 return undefined; 46 51 } 52 + 53 + let playlist: string; 54 + let thumbnail: string; 55 + 56 + if (videoMapping) { 57 + playlist = `${env.HLS_CDN_URL}/${videoMapping.bunnyGuid}/playlist.m3u8`; 58 + thumbnail = `${env.HLS_CDN_URL}/${videoMapping.bunnyGuid}/thumbnail.jpg`; 59 + } else { 60 + playlist = 61 + `${env.VIDEO_CDN_URL}/watch/${authorDid}/${embed.video.ref.$link}/playlist.m3u8`; 62 + thumbnail = 63 + `https://thumb.sprk.so/${authorDid}/${embed.video.ref.$link}/thumbnail`; 64 + } 65 + 47 66 return { 48 67 $type: "so.sprk.embed.video#view", 49 - cid, 68 + cid: embed.video.ref.$link, 50 69 alt: embed.alt, 51 - playlist: 52 - `https://media.sprk.so/video/${authorDid}/${embed.video.ref.$link}`, 53 - thumbnail: 54 - `https://thumb.sprk.so/${authorDid}/${embed.video.ref.$link}/thumbnail`, 70 + playlist, 71 + thumbnail, 55 72 } as const; 56 73 } 57 74 58 75 export function transformEmbed( 59 76 embed: PostEmbed | null, 60 77 authorDid: string, 61 - cid: string, 78 + videoMapping?: VideoMappingDocument | null, 62 79 options: ImageTransformOptions = {}, 63 80 ) { 64 81 if (!embed) { ··· 70 87 } 71 88 72 89 if (embed.$type === "so.sprk.embed.video") { 73 - return transformVideoEmbed(embed, authorDid, cid); 90 + return transformVideoEmbed(embed, authorDid, videoMapping); 74 91 } 75 92 76 93 return undefined;

+2

services/appview/utils/env.ts

··· 12 12 SERVICE_DID: envStr("SERVICE_DID") ?? "did:web:localhost", 13 13 MOD_SERVICE_DID: envStr("MOD_SERVICE_DID") ?? "did:web:localhost", 14 14 ADMIN_PASSWORD: envStr("ADMIN_PASSWORD") ?? "admin-token", 15 + HLS_CDN_URL: envStr("HLS_CDN_URL") ?? "https://vz-fb7436e9-c53.b-cdn.net", 16 + VIDEO_CDN_URL: envStr("VIDEO_CDN_URL") ?? "https://hls.sprk.so", 15 17 16 18 DB_URI: envStr("DB_URI"), 17 19 DB_NAME: envStr("DB_NAME") ?? "dev",

+161 -80

services/appview/utils/post-transformer.ts

··· 5 5 import { transformEmbed } from "./embed-transformer.ts"; 6 6 import { createProfileViewBasic } from "./profile-helper.ts"; 7 7 8 - // Transform DB post to PostView format 9 - export async function transformPostToPostView( 10 - post: PostDocument, 8 + // Transform DB posts to PostView format 9 + export async function transformPostsToPostViews( 10 + posts: PostDocument[], 11 11 db: Database, 12 12 userDid?: string, 13 - ): Promise<SoSprkFeedDefs.PostView> { 14 - // Get counts in parallel 15 - const [likeCount, replyCount, repostCount, lookCount, author] = await Promise 16 - .all([ 17 - // Get like count 18 - db.models.Like.countDocuments({ subject: post.uri }), 13 + ): Promise<SoSprkFeedDefs.PostView[]> { 14 + if (posts.length === 0) { 15 + return []; 16 + } 19 17 20 - // Get reply count 21 - db.models.Post.countDocuments({ 22 - "reply.parent.uri": post.uri, 23 - }), 18 + const postUris = posts.map((p) => p.uri); 19 + const authorDids = [...new Set(posts.map((p) => p.authorDid))]; 24 20 25 - // Get repost count 26 - db.models.Repost.countDocuments({ 27 - "subject.uri": post.uri, 21 + const [ 22 + likeCounts, 23 + replyCounts, 24 + repostCounts, 25 + lookCounts, 26 + authors, 27 + videoMappings, 28 + viewerLikes, 29 + viewerReposts, 30 + viewerLooks, 31 + ] = await Promise.all([ 32 + // Get like counts 33 + db.models.Like.aggregate([ 34 + { $match: { subject: { $in: postUris } } }, 35 + { $group: { _id: "$subject", count: { $sum: 1 } } }, 36 + ]), 37 + // Get reply counts 38 + db.models.Post.aggregate([ 39 + { $match: { "reply.parent.uri": { $in: postUris } } }, 40 + { $group: { _id: "$reply.parent.uri", count: { $sum: 1 } } }, 41 + ]), 42 + // Get repost counts 43 + db.models.Repost.aggregate([ 44 + { $match: { "subject.uri": { $in: postUris } } }, 45 + { $group: { _id: "$subject.uri", count: { $sum: 1 } } }, 46 + ]), 47 + // Get look counts 48 + db.models.Look.aggregate([ 49 + { $match: { "subject.uri": { $in: postUris } } }, 50 + { $group: { _id: "$subject.uri", count: { $sum: 1 } } }, 51 + ]), 52 + // Get authors 53 + Promise.all( 54 + authorDids.map(async (did) => { 55 + const author = await db.models.Profile.findOne({ authorDid: did }) 56 + .lean(); 57 + return createProfileViewBasic( 58 + did, 59 + author?.authorHandle || "unknown.invalid", 60 + db, 61 + ); 28 62 }), 63 + ), 64 + // Get video mappings 65 + db.models.VideoMapping.find({ 66 + key: { 67 + $in: posts 68 + .filter((p) => p.embed?.$type === "so.sprk.embed.video") 69 + .map((p) => `${p.authorDid}-${p.embed?.video?.ref.$link}`), 70 + }, 71 + }).lean(), 72 + // Get viewer likes 73 + userDid 74 + ? db.models.Like.find({ subject: { $in: postUris }, authorDid: userDid }) 75 + .lean() 76 + : Promise.resolve([]), 77 + // Get viewer reposts 78 + userDid 79 + ? db.models.Repost.find({ 80 + "subject.uri": { $in: postUris }, 81 + authorDid: userDid, 82 + }).lean() 83 + : Promise.resolve([]), 84 + // Get viewer looks 85 + userDid 86 + ? db.models.Look.find({ 87 + "subject.uri": { $in: postUris }, 88 + authorDid: userDid, 89 + }).lean() 90 + : Promise.resolve([]), 91 + ]); 29 92 30 - // Get look count 31 - db.models.Look.countDocuments({ 32 - "subject.uri": post.uri, 33 - }), 34 - 35 - // Create the author object with stories 36 - createProfileViewBasic(post.authorDid, post.authorHandle, db), 37 - ]); 38 - 39 - const embed = transformEmbed(post.embed, post.authorDid, post.cid); 93 + const likeCountsMap = new Map( 94 + likeCounts.map((item) => [item._id, item.count]), 95 + ); 96 + const replyCountsMap = new Map( 97 + replyCounts.map((item) => [item._id, item.count]), 98 + ); 99 + const repostCountsMap = new Map( 100 + repostCounts.map((item) => [item._id, item.count]), 101 + ); 102 + const lookCountsMap = new Map( 103 + lookCounts.map((item) => [item._id, item.count]), 104 + ); 105 + const authorsMap = new Map(authors.map((author) => [author.did, author])); 106 + const videoMappingsMap = new Map( 107 + videoMappings.map((item) => [item.key, item]), 108 + ); 109 + const viewerLikesMap = new Map( 110 + viewerLikes.map((like) => [like.subject, like.uri]), 111 + ); 112 + const viewerRepostsMap = new Map( 113 + viewerReposts.map((repost: { subject: { uri: string }; uri: string }) => [ 114 + repost.subject.uri, 115 + repost.uri, 116 + ]), 117 + ); 118 + const viewerLooksMap = new Map( 119 + viewerLooks.map((look: { subject: string; uri: string }) => [ 120 + look.subject, 121 + look.uri, 122 + ]), 123 + ); 40 124 41 - // Convert labels if any 42 - const labels = post.labels 43 - ? Array.isArray(post.labels) ? (post.labels as Label[]) : undefined 44 - : undefined; 125 + return posts.map((post) => { 126 + const videoMapping = post.embed?.$type === "so.sprk.embed.video" 127 + ? videoMappingsMap.get( 128 + `${post.authorDid}-${post.embed.video?.ref.$link}`, 129 + ) || null 130 + : null; 45 131 46 - // Build viewer state with information about the current user's interactions with the post 47 - const viewer: SoSprkFeedDefs.ViewerState = {}; 132 + const embed = transformEmbed( 133 + post.embed, 134 + post.authorDid, 135 + videoMapping, 136 + ); 48 137 49 - // Only check user interactions if a userDid is provided 50 - if (userDid) { 51 - // Check if the user has liked this post 52 - const like = await db.models.Like.findOne({ 53 - subject: post.uri, 54 - authorDid: userDid, 55 - }); 56 - if (like) { 57 - viewer.like = like.uri; 58 - } 138 + const labels = post.labels 139 + ? Array.isArray(post.labels) ? (post.labels as Label[]) : undefined 140 + : undefined; 59 141 60 - // Check if the user has reposted this post 61 - const repost = await db.models.Repost.findOne({ 62 - "subject.uri": post.uri, 63 - authorDid: userDid, 64 - }); 65 - if (repost) { 66 - viewer.repost = repost.uri; 142 + const viewer: SoSprkFeedDefs.ViewerState = {}; 143 + if (userDid) { 144 + viewer.like = viewerLikesMap.get(post.uri); 145 + viewer.repost = viewerRepostsMap.get(post.uri); 146 + viewer.look = viewerLooksMap.get(post.uri); 67 147 } 68 148 69 - // Check if the user has looked at this post 70 - const look = await db.models.Look.findOne({ 71 - "subject.uri": post.uri, 72 - authorDid: userDid, 73 - }); 74 - if (look) { 75 - viewer.look = look.uri; 76 - } 77 - } 149 + return { 150 + uri: post.uri, 151 + cid: post.cid, 152 + author: authorsMap.get(post.authorDid)!, 153 + record: { 154 + $type: "so.sprk.feed.post", 155 + text: post.text, 156 + embed: post.embed as SoSprkFeedPost.MainRecord["embed"], 157 + facets: post.facets, 158 + langs: post.langs, 159 + tags: post.tags, 160 + createdAt: post.createdAt, 161 + } satisfies SoSprkFeedPost.MainRecord, 162 + embed: embed, 163 + viewer, 164 + replyCount: replyCountsMap.get(post.uri) || 0, 165 + repostCount: repostCountsMap.get(post.uri) || 0, 166 + likeCount: likeCountsMap.get(post.uri) || 0, 167 + lookCount: lookCountsMap.get(post.uri) || 0, 168 + indexedAt: post.indexedAt, 169 + labels, 170 + }; 171 + }); 172 + } 78 173 79 - return { 80 - uri: post.uri, 81 - cid: post.cid, 82 - author, 83 - record: { 84 - $type: "so.sprk.feed.post", 85 - text: post.text, 86 - embed: post.embed as SoSprkFeedPost.MainRecord["embed"], 87 - facets: post.facets, 88 - langs: post.langs, 89 - tags: post.tags, 90 - createdAt: post.createdAt, 91 - } satisfies SoSprkFeedPost.MainRecord, 92 - embed: embed, 93 - viewer, 94 - replyCount, 95 - repostCount, 96 - likeCount, 97 - lookCount, 98 - indexedAt: post.indexedAt, 99 - labels, 100 - }; 174 + // Transform DB post to PostView format 175 + export async function transformPostToPostView( 176 + post: PostDocument, 177 + db: Database, 178 + userDid?: string, 179 + ): Promise<SoSprkFeedDefs.PostView> { 180 + const postViews = await transformPostsToPostViews([post], db, userDid); 181 + return postViews[0]; 101 182 }

+1 -1

services/appview/utils/story-transformer.ts

··· 15 15 db, 16 16 ); 17 17 18 - const embedView = transformEmbed(story.media, story.authorDid, story.cid, { 18 + const embedView = transformEmbed(story.media, story.authorDid, null, { 19 19 firstImageOnly: true, 20 20 }); 21 21

Configure Feed

Configure Feed