Feed a family of four (#39) · sprk.so/server@b49b935

+2 -1

README.md

··· 1 1 # Spark AppView 2 2 3 - This AppView provides a view of AT Protocol that encompasses all Spark lexicon and aims to interop with Bluesky lexicon. 3 + This AppView provides a view of AT Protocol that encompasses all Spark lexicon 4 + and aims to interop with Bluesky lexicon. 4 5 5 6 ## Development 6 7

+4 -3

api/com/atproto/admin/updateSubjectStatus.ts

··· 39 39 }); 40 40 await ctx.takedownService.updateRepoTakedownApplied( 41 41 repoRef.did, 42 - takedown.applied, 42 + true, 43 43 ); 44 44 } else { 45 45 await ctx.takedownService.removeRepoTakedown(repoRef.did); ··· 60 60 adminDid: auth.credentials.type === "standard" 61 61 ? auth.credentials.iss 62 62 : "admin", 63 + ref: takedown.ref, 63 64 }); 64 65 await ctx.takedownService.updateTakedownApplied( 65 66 recordRef.uri, 66 - takedown.applied, 67 + true, 67 68 ); 68 69 } else { 69 70 await ctx.takedownService.removeTakedown(recordRef.uri); ··· 89 90 await ctx.takedownService.updateBlobTakedownApplied( 90 91 repoBlobRef.did, 91 92 repoBlobRef.cid, 92 - takedown.applied, 93 + true, 93 94 ); 94 95 } else { 95 96 await ctx.takedownService.removeBlobTakedown(

+1 -6

api/com/atproto/repo/getRecord.ts

··· 63 63 } 64 64 65 65 // Parse the original record JSON 66 - let recordValue; 67 - try { 68 - recordValue = record.json ? JSON.parse(record.json) : record; 69 - } catch { 70 - throw new InvalidRequestError(`Invalid record JSON: ${uri}`); 71 - } 66 + const recordValue = record.json; 72 67 73 68 // Check if the record is subject to a takedown 74 69 const takedown = await ctx.takedownService.getTakedown(uri);

+4

api/index.ts

··· 18 18 import getStoriesTimeline from "./so/sprk/feed/getStoriesTimeline.ts"; 19 19 import getProfiles from "./so/sprk/actor/getProfiles.ts"; 20 20 import searchPosts from "./so/sprk/feed/searchPosts.ts"; 21 + import getSuggestedFeeds from "./so/sprk/feed/getSuggestedFeeds.ts"; 22 + import getTimeline from "./so/sprk/feed/getTimeline.ts"; 21 23 22 24 export default function (server: Server, ctx: AppContext) { 23 25 getAccountInfos(server, ctx); ··· 38 40 getStories(server, ctx); 39 41 getStoriesTimeline(server, ctx); 40 42 searchPosts(server, ctx); 43 + getSuggestedFeeds(server, ctx); 44 + getTimeline(server, ctx); 41 45 }

+5 -1

api/so/sprk/actor/getPreferences.ts

··· 1 1 import { Server } from "../../../../lex/index.ts"; 2 2 import { AppContext } from "../../../../main.ts"; 3 + import { Preferences } from "../../../../lex/types/so/sprk/actor/defs.ts"; 3 4 4 5 export default function (server: Server, ctx: AppContext) { 5 6 server.so.sprk.actor.getPreferences({ ··· 15 16 return { 16 17 encoding: "application/json", 17 18 body: { 18 - followMode: (userPref?.followMode || "sprk") as "sprk" | "bsky", 19 + preferences: [{ 20 + $type: "so.sprk.actor.defs#savedFeedsPref", 21 + items: (userPref?.savedFeeds ?? []), 22 + }] as Preferences, 19 23 }, 20 24 }; 21 25 } catch (error) {

+24 -26

api/so/sprk/actor/putPreferences.ts

··· 1 1 import { Server } from "../../../../lex/index.ts"; 2 + import { SavedFeedsPref } from "../../../../lex/types/so/sprk/actor/defs.ts"; 2 3 import { AppContext } from "../../../../main.ts"; 3 4 4 5 export default function (server: Server, ctx: AppContext) { ··· 7 8 handler: async ({ input, auth }) => { 8 9 const userDid = auth.credentials.iss; 9 10 const body = input.body; 10 - 11 - if (body.followMode && !["bsky", "sprk"].includes(body.followMode)) { 12 - throw new Error( 13 - 'Invalid followMode parameter. Must be "bsky" or "sprk"', 14 - ); 15 - } 16 11 17 12 try { 18 13 const now = new Date().toISOString(); 19 14 let userPref = await ctx.db.models.UserPreference.findOne({ userDid }); 20 - const oldMode = userPref?.followMode; 21 15 22 - if (!userPref) { 23 - userPref = await ctx.db.models.UserPreference.create({ 24 - userDid, 25 - createdAt: now, 26 - updatedAt: now, 27 - followMode: body.followMode || "sprk", // Default if not provided 28 - }); 29 - } else { 30 - if (body.followMode) { 31 - userPref.followMode = body.followMode; 32 - } 33 - userPref.updatedAt = now; 34 - await userPref.save(); 35 - } 16 + for (const pref of body.preferences) { 17 + if (pref as SavedFeedsPref) { 18 + const savedFeedsPref = pref as SavedFeedsPref; 36 19 37 - // Queue indexing of Bsky follows if switched to bsky mode 38 - if (body.followMode === "bsky" && oldMode !== "bsky") { 39 - ctx.sub.indexingSvc.indexRepo(userDid).catch((error) => 40 - ctx.logger.error("Failed to index repo", { error, userDid }) 41 - ); 20 + const savedFeeds = savedFeedsPref.items; 21 + 22 + if (!userPref) { 23 + userPref = await ctx.db.models.UserPreference.create({ 24 + userDid, 25 + savedFeeds: savedFeeds, 26 + createdAt: now, 27 + updatedAt: now, 28 + }); 29 + } else { 30 + await ctx.db.models.UserPreference.updateOne( 31 + { userDid }, 32 + { 33 + $push: { 34 + savedFeeds: { $each: savedFeeds }, 35 + }, 36 + }, 37 + ); 38 + } 39 + } 42 40 } 43 41 44 42 return;

+211

api/so/sprk/feed/getSuggestedFeeds.ts

··· 1 + import { Server } from "../../../../lex/index.ts"; 2 + import { AppContext } from "../../../../main.ts"; 3 + import { 4 + BskyGeneratorDocument, 5 + SprkGeneratorDocument, 6 + } from "../../../../data-plane/server/models.ts"; 7 + import { getProfileView } from "../../../../utils/profile-helper.ts"; 8 + import type * as SoSprkFeedDefs from "../../../../lex/types/so/sprk/feed/defs.ts"; 9 + import { decodeBase64, encodeBase64 } from "jsr:@std/encoding"; 10 + 11 + interface CursorData { 12 + likeCount: number; 13 + id: string; 14 + } 15 + 16 + // Helper function to parse cursor 17 + function parseCursor(cursor: string): CursorData { 18 + try { 19 + const decodedCursor = new TextDecoder().decode(decodeBase64(cursor)); 20 + const [likeCountStr, id] = decodedCursor.split("::"); 21 + 22 + if (!likeCountStr || !id) { 23 + throw new Error("Invalid cursor format"); 24 + } 25 + 26 + const likeCount = parseInt(likeCountStr, 10); 27 + if (isNaN(likeCount)) { 28 + throw new Error("Invalid cursor format"); 29 + } 30 + 31 + return { likeCount, id }; 32 + } catch { 33 + throw new Error("Invalid cursor format"); 34 + } 35 + } 36 + 37 + // Helper function to generate cursor 38 + function generateCursor(likeCount: number, id: string): string { 39 + return encodeBase64( 40 + new TextEncoder().encode(`${likeCount}::${id}`), 41 + ); 42 + } 43 + 44 + // Transform GeneratorDocument to GeneratorView 45 + async function transformGeneratorToView( 46 + generator: BskyGeneratorDocument | SprkGeneratorDocument, 47 + ctx: AppContext, 48 + viewerDid?: string, 49 + ): Promise<SoSprkFeedDefs.GeneratorView> { 50 + // Create the creator profile view 51 + const creator = await getProfileView(ctx, generator.authorDid, viewerDid); 52 + 53 + // Handle viewer state if user is authenticated 54 + let viewer: SoSprkFeedDefs.GeneratorViewerState | undefined; 55 + if (viewerDid) { 56 + const like = await ctx.db.models.Like.findOne({ 57 + authorDid: viewerDid, 58 + subject: generator.uri, 59 + }).lean(); 60 + 61 + if (like) { 62 + viewer = { 63 + $type: "so.sprk.feed.defs#generatorViewerState", 64 + like: like.uri, 65 + }; 66 + } 67 + } 68 + 69 + return { 70 + $type: "so.sprk.feed.defs#generatorView", 71 + uri: generator.uri, 72 + cid: generator.cid, 73 + did: generator.authorDid, 74 + creator, 75 + displayName: generator.displayName, 76 + description: generator.description || undefined, 77 + descriptionFacets: generator.descriptionFacets || undefined, 78 + avatar: generator.avatar?.ref?.$link 79 + ? `https://media.sprk.so/avatar/tiny/${generator.authorDid}/${generator.avatar.ref.$link}/webp` 80 + : undefined, 81 + likeCount: generator.likeCount || 0, 82 + acceptsInteractions: generator.acceptsInteractions || undefined, 83 + labels: undefined, // Labels will be handled separately if needed 84 + viewer, 85 + indexedAt: generator.indexedAt, 86 + }; 87 + } 88 + 89 + export default function (server: Server, ctx: AppContext) { 90 + server.so.sprk.feed.getSuggestedFeeds({ 91 + auth: ctx.authVerifier.standardOptional, 92 + handler: async ({ params, auth }) => { 93 + try { 94 + const { limit = 50, cursor } = params; 95 + const userDid = auth.credentials.type === "standard" 96 + ? auth.credentials.iss 97 + : undefined; 98 + 99 + // Validate limit 100 + if (limit < 1 || limit > 100) { 101 + throw new Error("Limit must be between 1 and 100"); 102 + } 103 + 104 + // Parse cursor if provided 105 + let cursorData: CursorData | undefined; 106 + if (cursor) { 107 + cursorData = parseCursor(cursor); 108 + } 109 + 110 + // Build query for generators sorted by like count 111 + const query: Record<string, unknown> = {}; 112 + 113 + // Add cursor-based pagination 114 + if (cursorData) { 115 + query.$or = [ 116 + { likeCount: { $lt: cursorData.likeCount } }, 117 + { likeCount: cursorData.likeCount, _id: { $lt: cursorData.id } }, 118 + ]; 119 + } 120 + 121 + // Get both BskyGenerator and SprkGenerator documents 122 + const [bskyGenerators, sprkGenerators] = await Promise.all([ 123 + ctx.db.models.BskyGenerator.find(query) 124 + .sort({ likeCount: -1, _id: -1 }) 125 + .lean(), 126 + ctx.db.models.SprkGenerator.find(query) 127 + .sort({ likeCount: -1, _id: -1 }) 128 + .lean(), 129 + ]); 130 + 131 + // Combine and sort all generators by like count 132 + const allGenerators = [...bskyGenerators, ...sprkGenerators] 133 + .sort((a, b) => { 134 + const aLikes = a.likeCount || 0; 135 + const bLikes = b.likeCount || 0; 136 + if (aLikes !== bLikes) { 137 + return bLikes - aLikes; // Sort by like count descending 138 + } 139 + // If like counts are equal, sort by _id descending for consistency 140 + return String(b._id).localeCompare(String(a._id)); 141 + }); 142 + 143 + // Apply limit and check for more results 144 + const generators = allGenerators.slice(0, limit + 1); 145 + 146 + // Check if there are more results 147 + const hasMore = generators.length > limit; 148 + if (hasMore) { 149 + generators.pop(); // Remove the extra item 150 + } 151 + 152 + // Transform generators to GeneratorView format 153 + const generatorViews = await Promise.all( 154 + generators.map((generator) => 155 + transformGeneratorToView(generator, ctx, userDid) 156 + ), 157 + ); 158 + 159 + // Generate next cursor if there are more results 160 + let nextCursor: string | undefined; 161 + if (hasMore && generators.length > 0) { 162 + const lastGenerator = generators[generators.length - 1]; 163 + nextCursor = generateCursor( 164 + lastGenerator.likeCount || 0, 165 + String(lastGenerator._id), 166 + ); 167 + } 168 + 169 + // Prepare response 170 + const response: { 171 + feeds: SoSprkFeedDefs.GeneratorView[]; 172 + cursor?: string; 173 + } = { 174 + feeds: generatorViews, 175 + }; 176 + 177 + if (nextCursor) { 178 + response.cursor = nextCursor; 179 + } 180 + 181 + return { 182 + encoding: "application/json", 183 + body: response, 184 + }; 185 + } catch (error) { 186 + // Handle specific error cases 187 + if (error instanceof Error) { 188 + const message = error.message; 189 + 190 + if (message.includes("cursor") || message.includes("Cursor")) { 191 + return { 192 + status: 400, 193 + message: "The provided cursor is invalid", 194 + }; 195 + } 196 + 197 + if (message.includes("limit") || message.includes("Limit")) { 198 + return { 199 + status: 400, 200 + message: "Limit must be between 1 and 100", 201 + }; 202 + } 203 + } 204 + 205 + // Log unexpected errors and rethrow 206 + console.error("Unexpected error in getSuggestedFeeds:", error); 207 + throw error; 208 + } 209 + }, 210 + }); 211 + }

+169

api/so/sprk/feed/getTimeline.ts

··· 1 + import { Server } from "../../../../lex/index.ts"; 2 + import { AppContext } from "../../../../main.ts"; 3 + import { transformPostsToPostViews } from "../../../../utils/post-transformer.ts"; 4 + import { decodeBase64, encodeBase64 } from "jsr:@std/encoding"; 5 + import { OutputSchema } from "../../../../lex/types/so/sprk/feed/getTimeline.ts"; 6 + 7 + interface CursorData { 8 + createdAt: string; 9 + id: string; 10 + } 11 + 12 + // Helper function to parse cursor 13 + function parseCursor(cursor: string): CursorData { 14 + try { 15 + const decodedCursor = new TextDecoder().decode(decodeBase64(cursor)); 16 + const [timestamp, id] = decodedCursor.split("::"); 17 + 18 + if (!timestamp || !id) { 19 + throw new Error("Invalid cursor format"); 20 + } 21 + 22 + return { createdAt: timestamp, id }; 23 + } catch { 24 + throw new Error("Invalid cursor format"); 25 + } 26 + } 27 + 28 + // Helper function to generate cursor 29 + function generateCursor(createdAt: string, id: string): string { 30 + return encodeBase64( 31 + new TextEncoder().encode(`${createdAt}::${id}`), 32 + ); 33 + } 34 + 35 + // Helper function to get followed user DIDs 36 + async function getFollowedUsers( 37 + ctx: AppContext, 38 + userDid: string, 39 + ): Promise<string[]> { 40 + const follows = await ctx.db.models.Follow.find({ 41 + authorDid: userDid, 42 + }).select("subject").lean(); 43 + 44 + return follows.map((follow) => follow.subject); 45 + } 46 + 47 + // Helper function to build timeline query 48 + function buildTimelineQuery( 49 + followedDids: string[], 50 + cursor?: CursorData, 51 + ): Record<string, unknown> { 52 + const query: Record<string, unknown> = { 53 + authorDid: { $in: followedDids }, 54 + reply: null, // Only show top-level posts, not replies 55 + }; 56 + 57 + // Add cursor-based pagination 58 + if (cursor) { 59 + query.$or = [ 60 + { createdAt: { $lt: cursor.createdAt } }, 61 + { createdAt: cursor.createdAt, _id: { $lt: cursor.id } }, 62 + ]; 63 + } 64 + 65 + return query; 66 + } 67 + 68 + export default function (server: Server, ctx: AppContext) { 69 + server.so.sprk.feed.getTimeline({ 70 + auth: ctx.authVerifier.standard, 71 + handler: async ({ params, auth }) => { 72 + try { 73 + const { limit = 50, cursor } = params; 74 + const userDid = auth.credentials.iss; 75 + 76 + // Validate limit 77 + if (limit < 1 || limit > 100) { 78 + throw new Error("Limit must be between 1 and 100"); 79 + } 80 + 81 + // Parse cursor if provided 82 + let cursorData: CursorData | undefined; 83 + if (cursor) { 84 + cursorData = parseCursor(cursor); 85 + } 86 + 87 + // Get list of users the authenticated user follows 88 + const followedDids = await getFollowedUsers(ctx, userDid); 89 + 90 + // If user doesn't follow anyone, return empty feed 91 + if (followedDids.length === 0) { 92 + return { 93 + encoding: "application/json", 94 + body: { 95 + feed: [], 96 + }, 97 + }; 98 + } 99 + 100 + // Build and execute query for posts from followed users 101 + const query = buildTimelineQuery(followedDids, cursorData); 102 + const posts = await ctx.db.models.Post.find(query) 103 + .sort({ createdAt: -1, _id: -1 }) 104 + .limit(limit + 1) // Get one extra for hasMore check 105 + .lean(); 106 + 107 + // Check if there are more results 108 + const hasMore = posts.length > limit; 109 + if (hasMore) { 110 + posts.pop(); // Remove the extra item 111 + } 112 + 113 + // Transform posts to feed view posts 114 + const feedViewPosts = await transformPostsToPostViews( 115 + posts, 116 + ctx, 117 + userDid, 118 + ); 119 + 120 + // Generate next cursor if there are more results 121 + let nextCursor: string | undefined; 122 + if (hasMore && posts.length > 0) { 123 + const lastPost = posts[posts.length - 1]; 124 + nextCursor = generateCursor( 125 + String(lastPost.createdAt), 126 + String(lastPost._id), 127 + ); 128 + } 129 + 130 + // Prepare response 131 + const response: OutputSchema = { 132 + feed: feedViewPosts.map((post) => ({ post })), 133 + }; 134 + 135 + if (nextCursor) { 136 + response.cursor = nextCursor; 137 + } 138 + 139 + return { 140 + encoding: "application/json", 141 + body: response, 142 + }; 143 + } catch (error) { 144 + // Handle specific error cases 145 + if (error instanceof Error) { 146 + const message = error.message; 147 + 148 + if (message.includes("cursor") || message.includes("Cursor")) { 149 + return { 150 + status: 400, 151 + message: "The provided cursor is invalid", 152 + }; 153 + } 154 + 155 + if (message.includes("limit") || message.includes("Limit")) { 156 + return { 157 + status: 400, 158 + message: "Limit must be between 1 and 100", 159 + }; 160 + } 161 + } 162 + 163 + // Log unexpected errors and rethrow 164 + console.error("Unexpected error in getTimeline:", error); 165 + throw error; 166 + } 167 + }, 168 + }); 169 + }

-7

api/so/sprk/graph/getFollowers.ts

··· 39 39 } 40 40 } 41 41 42 - const actorPref = await ctx.db.models.UserPreference.findOne({ 43 - userDid: actorDid, 44 - }); 45 - const actorFollowMode = actorPref?.followMode || "sprk"; 46 - 47 - // Build query 48 42 const query: RootFilterQuery<FollowDocument> = { 49 43 subject: actorDid, 50 - type: actorFollowMode, 51 44 }; 52 45 53 46 if (cursor) {

-7

api/so/sprk/graph/getFollows.ts

··· 39 39 } 40 40 } 41 41 42 - const actorPref = await ctx.db.models.UserPreference.findOne({ 43 - userDid: actorDid, 44 - }); 45 - const actorFollowMode = actorPref?.followMode || "sprk"; 46 - 47 - // Build query 48 42 const query: RootFilterQuery<FollowDocument> = { 49 43 authorDid: actorDid, 50 - type: actorFollowMode, 51 44 }; 52 45 53 46 if (cursor) {

+6 -6

data-plane/server/index.ts

··· 102 102 "Music", 103 103 models.musicSchema, 104 104 ), 105 - Look: this.connection.model<models.LookDocument>( 106 - "Look", 107 - models.lookSchema, 108 - ), 109 - Generator: this.connection.model<models.GeneratorDocument>( 105 + BskyGenerator: this.connection.model<models.BskyGeneratorDocument>( 110 106 "Generator", 111 - models.generatorSchema, 107 + models.bskyGeneratorSchema, 108 + ), 109 + SprkGenerator: this.connection.model<models.SprkGeneratorDocument>( 110 + "SprkGenerator", 111 + models.sprkGeneratorSchema, 112 112 ), 113 113 Takedown: this.connection.model<models.TakedownDocument>( 114 114 "Takedown",

+8 -5

data-plane/server/indexing/index.ts

··· 16 16 import { Database } from "../index.ts"; 17 17 import { ActorDocument } from "../models.ts"; 18 18 import * as Block from "./plugins/block.ts"; 19 - import * as Generator from "./plugins/generator.ts"; 19 + import * as Generator from "./plugins/generator/index.ts"; 20 20 import * as Follow from "./plugins/follow.ts"; 21 21 import * as Like from "./plugins/like.ts"; 22 22 import * as Post from "./plugins/post.ts"; ··· 36 36 follow: Follow.PluginType; 37 37 profile: Profile.PluginType; 38 38 block: Block.PluginType; 39 - generator: Generator.PluginType; 39 + bskyGenerator: Generator.Bsky.PluginType; 40 + sprkGenerator: Generator.Sprk.PluginType; 40 41 story: Story.PluginType; 41 42 audio: Audio.PluginType; 42 43 music: Music.PluginType; ··· 55 56 follow: Follow.makePlugin(this.db, this.background), 56 57 profile: Profile.makePlugin(this.db, this.background), 57 58 block: Block.makePlugin(this.db, this.background), 58 - generator: Generator.makePlugin(this.db, this.background), 59 + bskyGenerator: Generator.Bsky.makePlugin(this.db, this.background), 60 + sprkGenerator: Generator.Sprk.makePlugin(this.db, this.background), 59 61 story: Story.makePlugin(this.db, this.background), 60 62 audio: Audio.makePlugin(this.db, this.background), 61 63 music: Music.makePlugin(this.db, this.background), ··· 245 247 const indexers = Object.values( 246 248 this.records as Record<string, RecordProcessor<unknown, unknown>>, 247 249 ); 248 - return indexers.find((indexer) => indexer.collections.includes(collection)); 250 + return indexers.find((indexer) => indexer.collection === collection); 249 251 } 250 252 251 253 async updateActorStatus(did: string, active: boolean, status: string = "") { ··· 301 303 await this.db.models.Follow.deleteMany({ authorDid: did }); 302 304 await this.db.models.Repost.deleteMany({ authorDid: did }); 303 305 await this.db.models.Like.deleteMany({ authorDid: did }); 304 - await this.db.models.Generator.deleteMany({ authorDid: did }); 306 + await this.db.models.BskyGenerator.deleteMany({ authorDid: did }); 307 + await this.db.models.SprkGenerator.deleteMany({ authorDid: did }); 305 308 await this.db.models.Story.deleteMany({ authorDid: did }); 306 309 await this.db.models.Audio.deleteMany({ authorDid: did }); 307 310 await this.db.models.Music.deleteMany({ authorDid: did });

+2 -2

data-plane/server/indexing/plugins/audio.ts

··· 8 8 import { RecordProcessor } from "../processor.ts"; 9 9 import { normalizeObject } from "../../../../utils/embed-normalizer.ts"; 10 10 11 - const lexIds = [lex.ids.SoSprkFeedAudio]; 11 + const lexId = lex.ids.SoSprkFeedAudio; 12 12 type IndexedAudio = AudioDocument; 13 13 14 14 const insertFn = async ( ··· 81 81 background: BackgroundQueue, 82 82 ): PluginType => { 83 83 return new RecordProcessor(db, background, { 84 - lexIds, 84 + lexId, 85 85 insertFn, 86 86 findDuplicate, 87 87 deleteFn,

+2 -2

data-plane/server/indexing/plugins/block.ts

··· 7 7 import { BlockDocument } from "../../models.ts"; 8 8 import { RecordProcessor } from "../processor.ts"; 9 9 10 - const lexIds = [lex.ids.AppBskyGraphBlock]; 10 + const lexId = lex.ids.AppBskyGraphBlock; 11 11 type IndexedBlock = BlockDocument; 12 12 13 13 const insertFn = async ( ··· 94 94 background: BackgroundQueue, 95 95 ): PluginType => { 96 96 return new RecordProcessor(db, background, { 97 - lexIds, 97 + lexId, 98 98 insertFn, 99 99 findDuplicate, 100 100 deleteFn,

+6 -17

data-plane/server/indexing/plugins/follow.ts

··· 1 1 import { CID } from "multiformats/cid"; 2 2 import { AtUri, normalizeDatetimeAlways } from "@atproto/syntax"; 3 3 import * as lex from "../../../../lex/lexicons.ts"; 4 - import * as BskyFollow from "../../../../lex/types/app/bsky/graph/follow.ts"; 5 - import * as SprkFollow from "../../../../lex/types/so/sprk/graph/follow.ts"; 4 + import * as Follow from "../../../../lex/types/app/bsky/graph/follow.ts"; 6 5 import { BackgroundQueue } from "../../background.ts"; 7 6 import { Database } from "../../index.ts"; 8 7 import { FollowDocument } from "../../models.ts"; 9 8 import { RecordProcessor } from "../processor.ts"; 10 9 11 - const lexIds = [lex.ids.AppBskyGraphFollow, lex.ids.SoSprkGraphFollow]; 10 + const lexId = lex.ids.AppBskyGraphFollow; 12 11 type IndexedFollow = FollowDocument; 13 12 14 - // Union type for both follow record types 15 - type FollowRecord = BskyFollow.Record | SprkFollow.Record; 16 - 17 13 const insertFn = async ( 18 14 db: Database, 19 15 uri: AtUri, 20 16 cid: CID, 21 - obj: FollowRecord, 17 + obj: Follow.Record, 22 18 timestamp: string, 23 19 ): Promise<IndexedFollow | null> => { 24 - const followType = uri.collection === "app.bsky.graph.follow" 25 - ? "bsky" 26 - : "sprk"; 27 - 28 20 const follow = { 29 21 uri: uri.toString(), 30 22 cid: cid.toString(), ··· 32 24 subject: obj.subject, 33 25 createdAt: normalizeDatetimeAlways(obj.createdAt), 34 26 indexedAt: timestamp, 35 - type: followType as "bsky" | "sprk", 36 27 }; 37 28 38 29 // Use findOneAndUpdate with upsert on the compound key to handle potential duplicate key errors ··· 41 32 { 42 33 authorDid: follow.authorDid, 43 34 subject: follow.subject, 44 - type: follow.type, 45 35 }, 46 36 follow, 47 37 { upsert: true, new: true }, ··· 59 49 const findDuplicate = async ( 60 50 db: Database, 61 51 uri: AtUri, 62 - obj: FollowRecord, 52 + obj: Follow.Record, 63 53 ): Promise<AtUri | null> => { 64 54 const found = await db.models.Follow.findOne({ 65 55 authorDid: uri.host, 66 56 subject: obj.subject, 67 - type: uri.collection === "app.bsky.graph.follow" ? "bsky" : "sprk", 68 57 }).select("uri").lean(); 69 58 return found ? new AtUri(found.uri) : null; 70 59 }; ··· 146 135 } 147 136 }; 148 137 149 - export type PluginType = RecordProcessor<FollowRecord, IndexedFollow>; 138 + export type PluginType = RecordProcessor<Follow.Record, IndexedFollow>; 150 139 151 140 export const makePlugin = ( 152 141 db: Database, 153 142 background: BackgroundQueue, 154 143 ): PluginType => { 155 144 return new RecordProcessor(db, background, { 156 - lexIds, 145 + lexId, 157 146 insertFn, 158 147 findDuplicate, 159 148 deleteFn,

+11 -12

data-plane/server/indexing/plugins/generator.ts data-plane/server/indexing/plugins/generator/sprk.ts

··· 1 1 import { CID } from "multiformats/cid"; 2 2 import { AtUri, normalizeDatetimeAlways } from "@atproto/syntax"; 3 - import * as lex from "../../../../lex/lexicons.ts"; 4 - import * as FeedGenerator from "../../../../lex/types/app/bsky/feed/generator.ts"; 5 - import { BackgroundQueue } from "../../background.ts"; 6 - import { Database } from "../../index.ts"; 7 - import { GeneratorDocument } from "../../models.ts"; 8 - import { RecordProcessor } from "../processor.ts"; 3 + import * as lex from "../../../../../lex/lexicons.ts"; 4 + import * as FeedGenerator from "../../../../../lex/types/so/sprk/feed/generator.ts"; 5 + import { BackgroundQueue } from "../../../background.ts"; 6 + import { Database } from "../../../index.ts"; 7 + import { SprkGeneratorDocument } from "../../../models.ts"; 8 + import { RecordProcessor } from "../../processor.ts"; 9 9 10 - const lexIds = [lex.ids.AppBskyFeedGenerator]; 11 - type IndexedFeedGenerator = GeneratorDocument; 10 + const lexId = lex.ids.SoSprkFeedGenerator; 11 + type IndexedFeedGenerator = SprkGeneratorDocument; 12 12 13 13 const insertFn = async ( 14 14 db: Database, ··· 36 36 avatar, 37 37 acceptsInteractions: obj.acceptsInteractions || null, 38 38 labels: null, // Will be populated by label processing 39 - contentMode: null, // Not used in Bluesky 40 39 createdAt: normalizeDatetimeAlways(obj.createdAt), 41 40 indexedAt: timestamp, 42 41 }; 43 42 44 43 // Use findOneAndUpdate with upsert to handle potential duplicate key errors 45 44 try { 46 - const insertedGenerator = await db.models.Generator.findOneAndUpdate( 45 + const insertedGenerator = await db.models.SprkGenerator.findOneAndUpdate( 47 46 { uri: generator.uri }, 48 47 generator, 49 48 { upsert: true, new: true }, ··· 71 70 db: Database, 72 71 uri: AtUri, 73 72 ): Promise<IndexedFeedGenerator | null> => { 74 - const deleted = await db.models.Generator.findOneAndDelete({ 73 + const deleted = await db.models.SprkGenerator.findOneAndDelete({ 75 74 uri: uri.toString(), 76 75 }); 77 76 return deleted; ··· 91 90 background: BackgroundQueue, 92 91 ): PluginType => { 93 92 return new RecordProcessor(db, background, { 94 - lexIds, 93 + lexId, 95 94 insertFn, 96 95 findDuplicate, 97 96 deleteFn,

+103

data-plane/server/indexing/plugins/generator/bsky.ts

··· 1 + import { CID } from "multiformats/cid"; 2 + import { AtUri, normalizeDatetimeAlways } from "@atproto/syntax"; 3 + import * as lex from "../../../../../lex/lexicons.ts"; 4 + import * as FeedGenerator from "../../../../../lex/types/app/bsky/feed/generator.ts"; 5 + import { BackgroundQueue } from "../../../background.ts"; 6 + import { Database } from "../../../index.ts"; 7 + import { BskyGeneratorDocument } from "../../../models.ts"; 8 + import { RecordProcessor } from "../../processor.ts"; 9 + 10 + const lexId = lex.ids.AppBskyFeedGenerator; 11 + type IndexedFeedGenerator = BskyGeneratorDocument; 12 + 13 + const insertFn = async ( 14 + db: Database, 15 + uri: AtUri, 16 + cid: CID, 17 + obj: FeedGenerator.Record, 18 + timestamp: string, 19 + ): Promise<IndexedFeedGenerator | null> => { 20 + // Extract and clean avatar to ensure it matches MediaRef format 21 + const avatar = obj.avatar 22 + ? { 23 + $type: "blob", 24 + ref: (obj.avatar as unknown as Record<string, unknown>)?.ref || null, 25 + } 26 + : null; 27 + 28 + const generator = { 29 + uri: uri.toString(), 30 + cid: cid.toString(), 31 + authorDid: uri.host, 32 + did: obj.did, 33 + displayName: obj.displayName, 34 + description: obj.description || null, 35 + descriptionFacets: obj.descriptionFacets || null, 36 + avatar, 37 + acceptsInteractions: obj.acceptsInteractions || null, 38 + labels: null, // Will be populated by label processing 39 + contentMode: obj.contentMode, 40 + createdAt: normalizeDatetimeAlways(obj.createdAt), 41 + indexedAt: timestamp, 42 + }; 43 + 44 + // Use findOneAndUpdate with upsert to handle potential duplicate key errors 45 + try { 46 + const insertedGenerator = await db.models.BskyGenerator.findOneAndUpdate( 47 + { uri: generator.uri }, 48 + generator, 49 + { upsert: true, new: true }, 50 + ); 51 + return insertedGenerator; 52 + } catch (err) { 53 + // Handle duplicate key errors gracefully 54 + const mongoError = err as { code?: number }; 55 + if (mongoError.code === 11000) { 56 + return null; // Silently skip duplicates 57 + } 58 + throw err; 59 + } 60 + }; 61 + 62 + const findDuplicate = (): AtUri | null => { 63 + return null; 64 + }; 65 + 66 + const notifsForInsert = () => { 67 + return []; 68 + }; 69 + 70 + const deleteFn = async ( 71 + db: Database, 72 + uri: AtUri, 73 + ): Promise<IndexedFeedGenerator | null> => { 74 + const deleted = await db.models.BskyGenerator.findOneAndDelete({ 75 + uri: uri.toString(), 76 + }); 77 + return deleted; 78 + }; 79 + 80 + const notifsForDelete = () => { 81 + return { notifs: [], toDelete: [] }; 82 + }; 83 + 84 + export type PluginType = RecordProcessor< 85 + FeedGenerator.Record, 86 + IndexedFeedGenerator 87 + >; 88 + 89 + export const makePlugin = ( 90 + db: Database, 91 + background: BackgroundQueue, 92 + ): PluginType => { 93 + return new RecordProcessor(db, background, { 94 + lexId, 95 + insertFn, 96 + findDuplicate, 97 + deleteFn, 98 + notifsForInsert, 99 + notifsForDelete, 100 + }); 101 + }; 102 + 103 + export default makePlugin;

+2

data-plane/server/indexing/plugins/generator/index.ts

··· 1 + export * as Bsky from "./bsky.ts"; 2 + export * as Sprk from "./sprk.ts";

+44 -15

data-plane/server/indexing/plugins/like.ts

··· 7 7 import { LikeDocument } from "../../models.ts"; 8 8 import { RecordProcessor } from "../processor.ts"; 9 9 10 - const lexIds = [lex.ids.SoSprkFeedLike]; 10 + const lexId = lex.ids.SoSprkFeedLike; 11 11 type IndexedLike = LikeDocument; 12 12 13 13 const insertFn = async ( ··· 139 139 140 140 const updateAggregates = async (db: Database, like: IndexedLike) => { 141 141 try { 142 - // Update like count for the subject (count both types) 142 + // Update like count for the subject 143 143 const likeCount = await db.models.Like.countDocuments({ 144 144 subject: like.subject, 145 145 }); 146 146 147 - // First check if post exists to avoid creating one with missing fields 148 - const existingPost = await db.models.Post.findOne({ 149 - uri: like.subject, 150 - }); 147 + const subjectUri = new AtUri(like.subject); 148 + 149 + // Check if this is a feed generator 150 + if (subjectUri.collection === "app.bsky.feed.generator") { 151 + const existingGenerator = await db.models.BskyGenerator.findOne({ 152 + uri: like.subject, 153 + }); 154 + 155 + if (existingGenerator) { 156 + await db.models.BskyGenerator.findOneAndUpdate( 157 + { uri: like.subject }, 158 + { $set: { likeCount } }, 159 + { new: true }, 160 + ); 161 + } 162 + } else if (subjectUri.collection === "so.sprk.feed.generator") { 163 + const existingSprkGenerator = await db.models.SprkGenerator.findOne({ 164 + uri: like.subject, 165 + }); 166 + 167 + if (existingSprkGenerator) { 168 + await db.models.SprkGenerator.findOneAndUpdate( 169 + { uri: like.subject }, 170 + { $set: { likeCount } }, 171 + { new: true }, 172 + ); 173 + } 174 + } else { 175 + // Handle posts and other content types 176 + const existingPost = await db.models.Post.findOne({ 177 + uri: like.subject, 178 + }); 151 179 152 - if (existingPost) { 153 - // Only update existing posts 154 - await db.models.Post.findOneAndUpdate( 155 - { uri: like.subject }, 156 - { $set: { likeCount } }, 157 - { new: true }, 158 - ); 180 + if (existingPost) { 181 + // Only update existing posts 182 + await db.models.Post.findOneAndUpdate( 183 + { uri: like.subject }, 184 + { $set: { likeCount } }, 185 + { new: true }, 186 + ); 187 + } 188 + // We don't create a post if it doesn't exist, as we might lack required fields 159 189 } 160 - // We don't create a post if it doesn't exist, as we might lack required fields 161 190 } catch (error) { 162 191 console.error("Error updating like aggregates:", error); 163 192 // Don't throw - allow processing to continue even if aggregates update fails ··· 171 200 background: BackgroundQueue, 172 201 ): PluginType => { 173 202 return new RecordProcessor(db, background, { 174 - lexIds, 203 + lexId, 175 204 insertFn, 176 205 findDuplicate, 177 206 deleteFn,

+2 -2

data-plane/server/indexing/plugins/music.ts

··· 8 8 import { RecordProcessor } from "../processor.ts"; 9 9 import { normalizeObject } from "../../../../utils/embed-normalizer.ts"; 10 10 11 - const lexIds = [lex.ids.SoSprkFeedMusic]; 11 + const lexId = lex.ids.SoSprkFeedMusic; 12 12 type IndexedMusic = MusicDocument; 13 13 14 14 const insertFn = async ( ··· 85 85 background: BackgroundQueue, 86 86 ): PluginType => { 87 87 return new RecordProcessor(db, background, { 88 - lexIds, 88 + lexId, 89 89 insertFn, 90 90 findDuplicate, 91 91 deleteFn,

+5 -5

data-plane/server/indexing/plugins/post.ts

··· 1 1 import { CID } from "multiformats/cid"; 2 - import { jsonStringToLex } from "@atproto/lexicon"; 3 2 import { AtUri } from "@atproto/syntax"; 4 3 import * as lex from "../../../../lex/lexicons.ts"; 5 4 import { isMain as isEmbedImage } from "../../../../lex/types/so/sprk/embed/images.ts"; ··· 26 25 normalizeEmbed, 27 26 normalizeObject, 28 27 } from "../../../../utils/embed-normalizer.ts"; 28 + import { jsonToLex } from "@atproto/api"; 29 29 30 30 type PostAncestor = { 31 31 uri: string; ··· 59 59 threadgate?: GateRecord; 60 60 }; 61 61 62 - const lexIds = [lex.ids.SoSprkFeedPost]; 62 + const lexId = lex.ids.SoSprkFeedPost; 63 63 64 64 const REPLY_NOTIF_DEPTH = 5; 65 65 ··· 400 400 background: BackgroundQueue, 401 401 ): PluginType => { 402 402 return new RecordProcessor(db, background, { 403 - lexIds, 403 + lexId, 404 404 insertFn, 405 405 findDuplicate, 406 406 deleteFn, ··· 447 447 ? { 448 448 uri: root.uri, 449 449 invalidReplyRoot: root.invalidReplyRoot ?? null, 450 - record: jsonStringToLex(root.json) as PostRecord, 450 + record: jsonToLex(root.json) as PostRecord, 451 451 } 452 452 : null, 453 453 parent: parent && parent.json 454 454 ? { 455 455 uri: parent.uri, 456 456 invalidReplyRoot: parent.invalidReplyRoot ?? null, 457 - record: jsonStringToLex(parent.json) as PostRecord, 457 + record: jsonToLex(parent.json) as PostRecord, 458 458 } 459 459 : null, 460 460 };

+2 -2

data-plane/server/indexing/plugins/profile.ts

··· 8 8 import { RecordProcessor } from "../processor.ts"; 9 9 import { normalizeProfile } from "../../../../utils/embed-normalizer.ts"; 10 10 11 - const lexIds = [lex.ids.SoSprkActorProfile]; 11 + const lexId = lex.ids.SoSprkActorProfile; 12 12 type IndexedProfile = ProfileDocument; 13 13 14 14 const insertFn = async ( ··· 84 84 background: BackgroundQueue, 85 85 ): PluginType => { 86 86 return new RecordProcessor(db, background, { 87 - lexIds, 87 + lexId, 88 88 insertFn, 89 89 findDuplicate, 90 90 deleteFn,

+2 -2

data-plane/server/indexing/plugins/repost.ts

··· 7 7 import { RepostDocument } from "../../models.ts"; 8 8 import { RecordProcessor } from "../processor.ts"; 9 9 10 - const lexIds = [lex.ids.AppBskyFeedRepost]; 10 + const lexId = lex.ids.AppBskyFeedRepost; 11 11 type IndexedRepost = RepostDocument; 12 12 13 13 const insertFn = async ( ··· 219 219 background: BackgroundQueue, 220 220 ): PluginType => { 221 221 return new RecordProcessor(db, background, { 222 - lexIds, 222 + lexId, 223 223 insertFn, 224 224 findDuplicate, 225 225 deleteFn,

+2 -2

data-plane/server/indexing/plugins/story.ts

··· 11 11 normalizeObject, 12 12 } from "../../../../utils/embed-normalizer.ts"; 13 13 14 - const lexIds = [lex.ids.SoSprkFeedStory]; 14 + const lexId = lex.ids.SoSprkFeedStory; 15 15 type IndexedStory = StoryDocument; 16 16 17 17 const insertFn = async ( ··· 80 80 background: BackgroundQueue, 81 81 ): PluginType => { 82 82 return new RecordProcessor(db, background, { 83 - lexIds, 83 + lexId, 84 84 insertFn, 85 85 findDuplicate, 86 86 deleteFn,

+41 -28

data-plane/server/indexing/processor.ts

··· 1 1 import { CID } from "multiformats/cid"; 2 - import { jsonStringToLex, stringifyLex } from "@atproto/lexicon"; 2 + import { stringifyLex } from "@atproto/lexicon"; 3 3 import { AtUri } from "@atproto/syntax"; 4 4 import { lexicons } from "../../../lex/lexicons.ts"; 5 5 import { BackgroundQueue } from "../background.ts"; 6 6 import { Database } from "../index.ts"; 7 + import { jsonToLex } from "@atproto/api"; 7 8 8 9 // @NOTE re: insertions and deletions. Due to how record updates are handled, 9 10 // (insertFn) should have the same effect as (insertFn -> deleteFn -> insertFn). 10 11 type RecordProcessorParams<T, S> = { 11 - lexIds: string[]; 12 + lexId: string; 12 13 insertFn: ( 13 14 db: Database, 14 15 uri: AtUri, ··· 41 42 }; 42 43 43 44 export class RecordProcessor<T, S> { 44 - collections: string[]; 45 + collection: string; 45 46 db: Database; 46 47 47 48 /** 48 - * RecordProcessor for handling multiple AT Protocol collections. 49 + * RecordProcessor for handling a single AT Protocol collection. 49 50 * 50 - * This processor can handle multiple lexIds for similar record types: 51 - * - Validates records directly against their URI collection (e.g., "app.bsky.graph.follow") 52 - * - Uses lexIds list for routing to determine which processor handles which collections 53 - * - Provides shared logic for similar concepts across different AT Protocol namespaces 51 + * This processor handles exactly one lexId: 52 + * - Validates records against the specific lexId (e.g., "app.bsky.graph.follow") 53 + * - Only processes records that match the exact collection NSID 54 + * - Rejects records from other collections, even similar ones 54 55 * 55 56 * Example usage: 56 57 * ```typescript 57 58 * const processor = new RecordProcessor(db, background, { 58 - * lexIds: ["app.bsky.graph.follow", "so.sprk.graph.follow"], 59 + * lexId: "app.bsky.graph.follow", 59 60 * // ... other params 60 61 * }); 61 62 * 62 - * // This will validate against "app.bsky.graph.follow" directly 63 + * // This will only process records with collection "app.bsky.graph.follow" 63 64 * await processor.insertRecord(uri, cid, obj, timestamp); 64 65 * ``` 65 66 */ ··· 69 70 private params: RecordProcessorParams<T, S>, 70 71 ) { 71 72 this.db = appDb; 72 - this.collections = this.params.lexIds; 73 + this.collection = this.params.lexId; 74 + } 75 + 76 + matchesCollection(uri: AtUri): boolean { 77 + return uri.collection === this.collection; 73 78 } 74 79 75 - matchesSchema(obj: unknown, collection: string): obj is T { 76 - // The collection IS the lexId - direct validation 80 + matchesSchema(obj: unknown): obj is T { 77 81 try { 78 - lexicons.assertValidRecord(collection, obj); 82 + lexicons.assertValidRecord(this.collection, obj); 79 83 return true; 80 84 } catch { 81 85 return false; 82 86 } 83 87 } 84 88 85 - assertValidRecord(obj: unknown, collection: string): asserts obj is T { 86 - // The collection IS the lexId - direct validation 89 + assertValidRecord(obj: unknown, uri: AtUri): asserts obj is T { 90 + if (!this.matchesCollection(uri)) { 91 + throw new Error( 92 + `Record collection mismatch: expected ${this.collection}, got ${uri.collection}`, 93 + ); 94 + } 87 95 try { 88 - lexicons.assertValidRecord(collection, obj); 96 + lexicons.assertValidRecord(this.collection, obj); 89 97 } catch (err) { 90 98 throw new Error( 91 - `Record validation failed for collection: ${collection}. Error: ${err}`, 99 + `Record validation failed for collection: ${this.collection}. Error: ${err}`, 92 100 ); 93 101 } 94 102 } 95 103 96 - // Helper method to get all available lexIds for debugging 97 - getAvailableLexIds(): string[] { 98 - return [...this.params.lexIds]; 104 + // Helper method to get the lexId this processor handles 105 + getLexId(): string { 106 + return this.collection; 99 107 } 100 108 101 109 async insertRecord( ··· 105 113 timestamp: string, 106 114 opts?: { disableNotifs?: boolean }, 107 115 ) { 108 - this.assertValidRecord(obj, uri.collection); 116 + this.assertValidRecord(obj, uri); 109 117 110 118 // Insert or update record 111 119 await this.db.models.Record.findOneAndUpdate( ··· 163 171 timestamp: string, 164 172 opts?: { disableNotifs?: boolean }, 165 173 ) { 166 - this.assertValidRecord(obj, uri.collection); 174 + this.assertValidRecord(obj, uri); 167 175 168 176 // Update record 169 177 await this.db.models.Record.findOneAndUpdate( ··· 212 220 } 213 221 this.aggregateOnCommit(inserted); 214 222 if (!opts?.disableNotifs) { 215 - await this.handleNotifs({ inserted, deleted }); 223 + this.handleNotifs({ inserted, deleted }); 216 224 } 217 225 } 218 226 ··· 247 255 return this.handleNotifs({ deleted }); 248 256 } 249 257 250 - const record = jsonStringToLex(recordDoc.json); 251 - if (!this.matchesSchema(record, new AtUri(found.uri).collection)) { 258 + const foundUri = new AtUri(found.uri); 259 + if (!this.matchesCollection(foundUri)) { 260 + return this.handleNotifs({ deleted }); 261 + } 262 + 263 + const record = jsonToLex(recordDoc.json); 264 + if (!this.matchesSchema(record)) { 252 265 return this.handleNotifs({ deleted }); 253 266 } 254 267 255 268 const inserted = await this.params.insertFn( 256 269 this.db, 257 - new AtUri(found.uri), 270 + foundUri, 258 271 CID.parse(found.cid), 259 272 record, 260 273 found.indexedAt, ··· 262 275 if (inserted) { 263 276 this.aggregateOnCommit(inserted); 264 277 } 265 - await this.handleNotifs({ deleted, inserted: inserted ?? undefined }); 278 + this.handleNotifs({ deleted, inserted: inserted ?? undefined }); 266 279 } 267 280 } 268 281

+58 -38

data-plane/server/models.ts

··· 86 86 rkey: string; 87 87 createdAt: string; 88 88 indexedAt: string; 89 - json?: string; 89 + json: JSON; 90 90 invalidReplyRoot?: boolean; 91 + takenDown: boolean; 92 + takedownRef: string; 91 93 } 92 94 93 95 export const recordSchema = new Schema<RecordDocument>({ ··· 98 100 rkey: { type: String, required: true }, 99 101 createdAt: { type: String, required: true }, 100 102 indexedAt: { type: String, required: true }, 101 - json: { type: String, required: false }, 103 + json: { type: JSON, required: true }, 102 104 invalidReplyRoot: { type: Boolean, required: false }, 105 + takenDown: { type: Boolean, required: false }, 106 + takedownRef: { type: String, required: false }, 103 107 }); 104 108 105 109 export interface DuplicateRecordDocument extends Document { ··· 143 147 viaCid: { type: String, required: false }, 144 148 }); 145 149 146 - export interface LookDocument extends AuthoredDocument { 147 - subject: string; 148 - subjectCid: string; 149 - cid: string; 150 - } 151 - 152 - export const lookSchema = new Schema<LookDocument>({ 153 - ...authoredSchema, 154 - subject: { type: String, required: true, index: true }, 155 - subjectCid: { type: String, required: true }, 156 - }); 157 - 158 150 export interface FollowDocument extends AuthoredDocument { 159 151 subject: string; 160 - type: "sprk" | "bsky"; 161 152 } 162 153 163 154 export const followSchema = new Schema<FollowDocument>({ 164 155 ...authoredSchema, 165 156 subject: { type: String, required: true, index: true }, 166 - type: { 167 - type: String, 168 - required: true, 169 - enum: ["sprk", "bsky"], 170 - index: true, 171 - default: "sprk", 172 - }, 173 157 }); 174 158 175 159 export interface BlockDocument extends AuthoredDocument { ··· 427 411 musicSchema.index({ authorDid: 1, createdAt: -1 }); 428 412 musicSchema.index({ tags: 1, createdAt: -1 }); 429 413 430 - export interface GeneratorDocument extends AuthoredDocument { 414 + export interface BskyGeneratorDocument extends AuthoredDocument { 431 415 displayName: string; 432 416 description?: string; 433 417 descriptionFacets?: Facet[]; 434 418 avatar?: MediaRef; 435 419 acceptsInteractions?: boolean; 420 + contentMode?: "video" | "unspecified"; 436 421 labels?: Label[]; 437 - contentMode?: string; 422 + likeCount: number; 438 423 } 439 424 440 - export const generatorSchema = new Schema<GeneratorDocument>({ 425 + export const bskyGeneratorSchema = new Schema<BskyGeneratorDocument>({ 441 426 ...authoredSchema, 442 427 displayName: { type: String, required: true }, 443 428 description: { type: String, required: false }, 444 429 descriptionFacets: { type: [Object], required: false }, 445 430 avatar: { type: Object, required: false }, 446 431 acceptsInteractions: { type: Boolean, required: false }, 432 + contentMode: { 433 + type: String, 434 + enum: ["video", "unspecified"], 435 + required: false, 436 + }, 447 437 labels: { type: [Object], required: false }, 448 - contentMode: { type: String, required: false }, 438 + likeCount: { type: Number, required: false, default: 0 }, 449 439 }); 450 440 451 441 // Add compound indexes for Generator 452 - generatorSchema.index({ authorDid: 1, createdAt: -1 }); 442 + bskyGeneratorSchema.index({ authorDid: 1, createdAt: -1 }); 443 + 444 + // @TODO: Currently this is almost identical to bskyGeneratorSchema but 445 + // as we make the feed lex meaningfully different from Bsky's feed lex, 446 + // we will add more fields and different behavior such as feed modes & 447 + // custom user values. 448 + export interface SprkGeneratorDocument extends AuthoredDocument { 449 + displayName: string; 450 + description?: string; 451 + descriptionFacets?: Facet[]; 452 + avatar?: MediaRef; 453 + acceptsInteractions?: boolean; 454 + labels?: Label[]; 455 + likeCount: number; 456 + } 457 + 458 + export const sprkGeneratorSchema = new Schema<SprkGeneratorDocument>({ 459 + ...authoredSchema, 460 + displayName: { type: String, required: true }, 461 + description: { type: String, required: false }, 462 + descriptionFacets: { type: [Object], required: false }, 463 + avatar: { type: Object, required: false }, 464 + acceptsInteractions: { type: Boolean, required: false }, 465 + labels: { type: [Object], required: false }, 466 + likeCount: { type: Number, required: false, default: 0 }, 467 + }); 468 + 469 + // Add compound indexes for Generator 470 + sprkGeneratorSchema.index({ authorDid: 1, createdAt: -1 }); 453 471 454 472 export interface TakedownDocument extends Document { 455 473 targetUri: string; ··· 534 552 services: { type: String, required: true }, 535 553 }); 536 554 555 + type SavedFeed = { 556 + id: string; 557 + type: "feed" | "list" | "timeline"; 558 + value: string; 559 + pinned: boolean; 560 + }; 561 + 537 562 export interface UserPreferenceDocument extends Document { 538 563 userDid: string; 539 - followMode: string; 564 + savedFeeds: SavedFeed[]; 540 565 createdAt: string; 541 566 updatedAt: string; 542 567 } 543 568 544 569 export const userPreferenceSchema = new Schema<UserPreferenceDocument>({ 545 570 userDid: { type: String, required: true, unique: true, index: true }, 546 - followMode: { 547 - type: String, 548 - required: true, 549 - enum: ["bsky", "sprk"], 550 - default: "sprk", 551 - }, 571 + savedFeeds: { type: [Object], required: true }, 552 572 createdAt: { type: String, required: true }, 553 573 updatedAt: { type: String, required: true }, 554 574 }); ··· 581 601 ([ 582 602 profileSchema, 583 603 likeSchema, 584 - lookSchema, 585 604 postSchema, 586 605 repostSchema, 587 606 followSchema, 588 607 blockSchema, 589 - generatorSchema, 608 + bskyGeneratorSchema, 609 + sprkGeneratorSchema, 590 610 audioSchema, 591 611 musicSchema, 592 612 storySchema, ··· 604 624 Audio: Model<AudioDocument>; 605 625 Repost: Model<RepostDocument>; 606 626 Music: Model<MusicDocument>; 607 - Look: Model<LookDocument>; 608 - Generator: Model<GeneratorDocument>; 627 + BskyGenerator: Model<BskyGeneratorDocument>; 628 + SprkGenerator: Model<SprkGeneratorDocument>; 609 629 Takedown: Model<TakedownDocument>; 610 630 RepoTakedown: Model<RepoTakedownDocument>; 611 631 BlobTakedown: Model<BlobTakedownDocument>;

+2 -4

data-plane/server/subscription.ts

··· 69 69 70 70 // Read fresh cursor from database 71 71 const savedCursor = await this.opts.db.getCursorState(); 72 - const startCursor = savedCursor !== null 73 - ? savedCursor 74 - : (env.NODE_ENV === "production" ? 0 : undefined); 72 + const startCursor = savedCursor !== null ? savedCursor : undefined; 75 73 76 74 const { runner, firehose } = createFirehose({ 77 75 idResolver: this.opts.idResolver, ··· 169 167 cursorSaveIntervalMs: 30000, // Save cursor every 30 seconds 170 168 setCursor: async (cursor: number) => { 171 169 await db.saveCursorState(cursor); 172 - logger.debug("Cursor saved to database", { cursor }); 170 + logger.info("Cursor saved to database", { cursor }); 173 171 }, 174 172 }); 175 173 const firehose = new Firehose({

-26

lex/index.ts

··· 167 167 import * as SoSprkFeedGetFeedGenerator from "./types/so/sprk/feed/getFeedGenerator.ts"; 168 168 import * as SoSprkFeedGetAuthorFeed from "./types/so/sprk/feed/getAuthorFeed.ts"; 169 169 import * as SoSprkFeedGetLikes from "./types/so/sprk/feed/getLikes.ts"; 170 - import * as SoSprkFeedGetActorLooks from "./types/so/sprk/feed/getActorLooks.ts"; 171 170 import * as SoSprkFeedGetPostThread from "./types/so/sprk/feed/getPostThread.ts"; 172 - import * as SoSprkFeedGetLooks from "./types/so/sprk/feed/getLooks.ts"; 173 171 import * as SoSprkFeedGetActorLikes from "./types/so/sprk/feed/getActorLikes.ts"; 174 172 import * as SoSprkFeedGetRepostedBy from "./types/so/sprk/feed/getRepostedBy.ts"; 175 173 import * as SoSprkFeedDescribeFeedGenerator from "./types/so/sprk/feed/describeFeedGenerator.ts"; ··· 2564 2562 return this._server.xrpc.method(nsid, cfg); 2565 2563 } 2566 2564 2567 - getActorLooks<A extends Auth = void>( 2568 - cfg: MethodConfigOrHandler< 2569 - A, 2570 - SoSprkFeedGetActorLooks.QueryParams, 2571 - SoSprkFeedGetActorLooks.HandlerInput, 2572 - SoSprkFeedGetActorLooks.HandlerOutput 2573 - >, 2574 - ) { 2575 - const nsid = "so.sprk.feed.getActorLooks"; // @ts-ignore - userType.nsid is dynamically generated and TypeScript can't infer its type 2576 - return this._server.xrpc.method(nsid, cfg); 2577 - } 2578 - 2579 2565 getPostThread<A extends Auth = void>( 2580 2566 cfg: MethodConfigOrHandler< 2581 2567 A, ··· 2585 2571 >, 2586 2572 ) { 2587 2573 const nsid = "so.sprk.feed.getPostThread"; // @ts-ignore - userType.nsid is dynamically generated and TypeScript can't infer its type 2588 - return this._server.xrpc.method(nsid, cfg); 2589 - } 2590 - 2591 - getLooks<A extends Auth = void>( 2592 - cfg: MethodConfigOrHandler< 2593 - A, 2594 - SoSprkFeedGetLooks.QueryParams, 2595 - SoSprkFeedGetLooks.HandlerInput, 2596 - SoSprkFeedGetLooks.HandlerOutput 2597 - >, 2598 - ) { 2599 - const nsid = "so.sprk.feed.getLooks"; // @ts-ignore - userType.nsid is dynamically generated and TypeScript can't infer its type 2600 2574 return this._server.xrpc.method(nsid, cfg); 2601 2575 } 2602 2576

+13 -253

lex/lexicons.ts

··· 13800 13800 "lex:com.atproto.label.defs#selfLabels", 13801 13801 ], 13802 13802 }, 13803 - "contentMode": { 13804 - "type": "string", 13805 - "knownValues": [ 13806 - "so.sprk.feed.defs#contentModeUnspecified", 13807 - "so.sprk.feed.defs#contentModeVideo", 13808 - ], 13809 - }, 13810 13803 "createdAt": { 13811 13804 "type": "string", 13812 13805 "format": "datetime", ··· 14055 14048 "likeCount": { 14056 14049 "type": "integer", 14057 14050 }, 14058 - "lookCount": { 14059 - "type": "integer", 14060 - }, 14061 14051 "indexedAt": { 14062 14052 "type": "string", 14063 14053 "format": "datetime", ··· 14133 14123 "format": "at-uri", 14134 14124 }, 14135 14125 "like": { 14136 - "type": "string", 14137 - "format": "at-uri", 14138 - }, 14139 - "look": { 14140 14126 "type": "string", 14141 14127 "format": "at-uri", 14142 14128 }, ··· 14418 14404 "type": "integer", 14419 14405 "minimum": 0, 14420 14406 }, 14421 - "lookCount": { 14422 - "type": "integer", 14423 - "minimum": 0, 14424 - }, 14425 14407 "acceptsInteractions": { 14426 14408 "type": "boolean", 14427 14409 }, ··· 14436 14418 "type": "ref", 14437 14419 "ref": "lex:so.sprk.feed.defs#generatorViewerState", 14438 14420 }, 14439 - "contentMode": { 14440 - "type": "string", 14441 - "knownValues": [ 14442 - "so.sprk.feed.defs#contentModeUnspecified", 14443 - "so.sprk.feed.defs#contentModeVideo", 14444 - ], 14445 - }, 14446 14421 "indexedAt": { 14447 14422 "type": "string", 14448 14423 "format": "datetime", ··· 14453 14428 "type": "object", 14454 14429 "properties": { 14455 14430 "like": { 14456 - "type": "string", 14457 - "format": "at-uri", 14458 - }, 14459 - "look": { 14460 14431 "type": "string", 14461 14432 "format": "at-uri", 14462 14433 }, ··· 14935 14906 }, 14936 14907 }, 14937 14908 }, 14938 - "SoSprkFeedGetActorLooks": { 14939 - "lexicon": 1, 14940 - "id": "so.sprk.feed.getActorLooks", 14941 - "defs": { 14942 - "main": { 14943 - "type": "query", 14944 - "description": 14945 - "Get a list of posts looked by an actor. Requires auth, actor must be the requesting account.", 14946 - "parameters": { 14947 - "type": "params", 14948 - "required": [ 14949 - "actor", 14950 - ], 14951 - "properties": { 14952 - "actor": { 14953 - "type": "string", 14954 - "format": "at-identifier", 14955 - }, 14956 - "limit": { 14957 - "type": "integer", 14958 - "minimum": 1, 14959 - "maximum": 100, 14960 - "default": 50, 14961 - }, 14962 - "cursor": { 14963 - "type": "string", 14964 - }, 14965 - }, 14966 - }, 14967 - "output": { 14968 - "encoding": "application/json", 14969 - "schema": { 14970 - "type": "object", 14971 - "required": [ 14972 - "feed", 14973 - ], 14974 - "properties": { 14975 - "cursor": { 14976 - "type": "string", 14977 - }, 14978 - "feed": { 14979 - "type": "array", 14980 - "items": { 14981 - "type": "ref", 14982 - "ref": "lex:so.sprk.feed.defs#feedViewPost", 14983 - }, 14984 - }, 14985 - }, 14986 - }, 14987 - }, 14988 - "errors": [ 14989 - { 14990 - "name": "BlockedActor", 14991 - }, 14992 - { 14993 - "name": "BlockedByActor", 14994 - }, 14995 - ], 14996 - }, 14997 - }, 14998 - }, 14999 14909 "SoSprkFeedPostgate": { 15000 14910 "lexicon": 1, 15001 14911 "id": "so.sprk.feed.postgate", ··· 15203 15113 }, 15204 15114 }, 15205 15115 }, 15206 - "SoSprkFeedGetLooks": { 15207 - "lexicon": 1, 15208 - "id": "so.sprk.feed.getLooks", 15209 - "defs": { 15210 - "main": { 15211 - "type": "query", 15212 - "description": 15213 - "Get look records which reference a subject (by AT-URI and CID).", 15214 - "parameters": { 15215 - "type": "params", 15216 - "required": [ 15217 - "uri", 15218 - ], 15219 - "properties": { 15220 - "uri": { 15221 - "type": "string", 15222 - "format": "at-uri", 15223 - "description": "AT-URI of the subject (eg, a post record).", 15224 - }, 15225 - "cid": { 15226 - "type": "string", 15227 - "format": "cid", 15228 - "description": 15229 - "CID of the subject record (aka, specific version of record), to filter looks.", 15230 - }, 15231 - "limit": { 15232 - "type": "integer", 15233 - "minimum": 1, 15234 - "maximum": 100, 15235 - "default": 50, 15236 - }, 15237 - "cursor": { 15238 - "type": "string", 15239 - }, 15240 - }, 15241 - }, 15242 - "output": { 15243 - "encoding": "application/json", 15244 - "schema": { 15245 - "type": "object", 15246 - "required": [ 15247 - "uri", 15248 - "looks", 15249 - ], 15250 - "properties": { 15251 - "uri": { 15252 - "type": "string", 15253 - "format": "at-uri", 15254 - }, 15255 - "cid": { 15256 - "type": "string", 15257 - "format": "cid", 15258 - }, 15259 - "cursor": { 15260 - "type": "string", 15261 - }, 15262 - "looks": { 15263 - "type": "array", 15264 - "items": { 15265 - "type": "ref", 15266 - "ref": "lex:so.sprk.feed.getLooks#look", 15267 - }, 15268 - }, 15269 - }, 15270 - }, 15271 - }, 15272 - }, 15273 - "look": { 15274 - "type": "object", 15275 - "required": [ 15276 - "indexedAt", 15277 - "createdAt", 15278 - "actor", 15279 - ], 15280 - "properties": { 15281 - "indexedAt": { 15282 - "type": "string", 15283 - "format": "datetime", 15284 - }, 15285 - "createdAt": { 15286 - "type": "string", 15287 - "format": "datetime", 15288 - }, 15289 - "actor": { 15290 - "type": "ref", 15291 - "ref": "lex:so.sprk.actor.defs#profileView", 15292 - }, 15293 - }, 15294 - }, 15295 - }, 15296 - }, 15297 15116 "SoSprkFeedGetActorLikes": { 15298 15117 "lexicon": 1, 15299 15118 "id": "so.sprk.feed.getActorLikes", ··· 15883 15702 }, 15884 15703 }, 15885 15704 }, 15886 - "SoSprkFeedLook": { 15887 - "lexicon": 1, 15888 - "id": "so.sprk.feed.look", 15889 - "defs": { 15890 - "main": { 15891 - "type": "record", 15892 - "description": 15893 - "Record declaring a 'look' of a piece of subject content. Equivalent to a 'view'", 15894 - "key": "tid", 15895 - "record": { 15896 - "type": "object", 15897 - "required": [ 15898 - "subject", 15899 - "createdAt", 15900 - ], 15901 - "properties": { 15902 - "subject": { 15903 - "type": "ref", 15904 - "ref": "lex:com.atproto.repo.strongRef", 15905 - }, 15906 - "createdAt": { 15907 - "type": "string", 15908 - "format": "datetime", 15909 - }, 15910 - }, 15911 - }, 15912 - }, 15913 - }, 15914 - }, 15915 15705 "SoSprkFeedGetQuotes": { 15916 15706 "lexicon": 1, 15917 15707 "id": "so.sprk.feed.getQuotes", ··· 16780 16570 "lex:so.sprk.actor.defs#adultContentPref", 16781 16571 "lex:so.sprk.actor.defs#contentLabelPref", 16782 16572 "lex:so.sprk.actor.defs#savedFeedsPref", 16783 - "lex:so.sprk.actor.defs#savedFeedsPrefV2", 16784 16573 "lex:so.sprk.actor.defs#personalDetailsPref", 16785 16574 "lex:so.sprk.actor.defs#feedViewPref", 16786 16575 "lex:so.sprk.actor.defs#threadViewPref", ··· 16859 16648 }, 16860 16649 }, 16861 16650 }, 16862 - "savedFeedsPrefV2": { 16651 + "savedFeedsPref": { 16863 16652 "type": "object", 16864 16653 "required": [ 16865 16654 "items", ··· 16874 16663 }, 16875 16664 }, 16876 16665 }, 16877 - "savedFeedsPref": { 16878 - "type": "object", 16879 - "required": [ 16880 - "pinned", 16881 - "saved", 16882 - ], 16883 - "properties": { 16884 - "pinned": { 16885 - "type": "array", 16886 - "items": { 16887 - "type": "string", 16888 - "format": "at-uri", 16889 - }, 16890 - }, 16891 - "saved": { 16892 - "type": "array", 16893 - "items": { 16894 - "type": "string", 16895 - "format": "at-uri", 16896 - }, 16897 - }, 16898 - "timelineIndex": { 16899 - "type": "integer", 16900 - }, 16901 - }, 16902 - }, 16903 16666 "personalDetailsPref": { 16904 16667 "type": "object", 16905 16668 "properties": { ··· 17139 16902 "encoding": "application/json", 17140 16903 "schema": { 17141 16904 "type": "object", 16905 + "required": [ 16906 + "preferences", 16907 + ], 17142 16908 "properties": { 17143 - "followMode": { 17144 - "type": "string", 17145 - "knownValues": [ 17146 - "bsky", 17147 - "sprk", 17148 - ], 16909 + "preferences": { 16910 + "type": "ref", 16911 + "ref": "lex:so.sprk.actor.defs#preferences", 17149 16912 }, 17150 16913 }, 17151 16914 }, ··· 17346 17109 "encoding": "application/json", 17347 17110 "schema": { 17348 17111 "type": "object", 17112 + "required": [ 17113 + "preferences", 17114 + ], 17349 17115 "properties": { 17350 - "followMode": { 17351 - "type": "string", 17352 - "knownValues": [ 17353 - "bsky", 17354 - "sprk", 17355 - ], 17116 + "preferences": { 17117 + "type": "ref", 17118 + "ref": "lex:so.sprk.actor.defs#preferences", 17356 17119 }, 17357 17120 }, 17358 17121 }, ··· 22491 22254 SoSprkFeedGetFeedGenerator: "so.sprk.feed.getFeedGenerator", 22492 22255 SoSprkFeedGetAuthorFeed: "so.sprk.feed.getAuthorFeed", 22493 22256 SoSprkFeedGetLikes: "so.sprk.feed.getLikes", 22494 - SoSprkFeedGetActorLooks: "so.sprk.feed.getActorLooks", 22495 22257 SoSprkFeedPostgate: "so.sprk.feed.postgate", 22496 22258 SoSprkFeedThreadgate: "so.sprk.feed.threadgate", 22497 22259 SoSprkFeedGetPostThread: "so.sprk.feed.getPostThread", 22498 - SoSprkFeedGetLooks: "so.sprk.feed.getLooks", 22499 22260 SoSprkFeedGetActorLikes: "so.sprk.feed.getActorLikes", 22500 22261 SoSprkFeedLike: "so.sprk.feed.like", 22501 22262 SoSprkFeedGetRepostedBy: "so.sprk.feed.getRepostedBy", ··· 22507 22268 SoSprkFeedGetFeed: "so.sprk.feed.getFeed", 22508 22269 SoSprkFeedGetStories: "so.sprk.feed.getStories", 22509 22270 SoSprkFeedAudio: "so.sprk.feed.audio", 22510 - SoSprkFeedLook: "so.sprk.feed.look", 22511 22271 SoSprkFeedGetQuotes: "so.sprk.feed.getQuotes", 22512 22272 SoSprkFeedGetStoriesTimeline: "so.sprk.feed.getStoriesTimeline", 22513 22273 SoSprkFeedGetFeedSkeleton: "so.sprk.feed.getFeedSkeleton",

+1 -19

lex/types/so/sprk/actor/defs.ts

··· 174 174 | $Typed<AdultContentPref> 175 175 | $Typed<ContentLabelPref> 176 176 | $Typed<SavedFeedsPref> 177 - | $Typed<SavedFeedsPrefV2> 178 177 | $Typed<PersonalDetailsPref> 179 178 | $Typed<FeedViewPref> 180 179 | $Typed<ThreadViewPref> ··· 246 245 return validate<SavedFeed & V>(v, id, hashSavedFeed); 247 246 } 248 247 249 - export interface SavedFeedsPrefV2 { 250 - $type?: "so.sprk.actor.defs#savedFeedsPrefV2"; 251 - items: (SavedFeed)[]; 252 - } 253 - 254 - const hashSavedFeedsPrefV2 = "savedFeedsPrefV2"; 255 - 256 - export function isSavedFeedsPrefV2<V>(v: V) { 257 - return is$typed(v, id, hashSavedFeedsPrefV2); 258 - } 259 - 260 - export function validateSavedFeedsPrefV2<V>(v: V) { 261 - return validate<SavedFeedsPrefV2 & V>(v, id, hashSavedFeedsPrefV2); 262 - } 263 - 264 248 export interface SavedFeedsPref { 265 249 $type?: "so.sprk.actor.defs#savedFeedsPref"; 266 - pinned: (string)[]; 267 - saved: (string)[]; 268 - timelineIndex?: number; 250 + items: (SavedFeed)[]; 269 251 } 270 252 271 253 const hashSavedFeedsPref = "savedFeedsPref";

+3 -4

lex/types/so/sprk/actor/getPreferences.ts

··· 1 1 /** 2 2 * GENERATED CODE - DO NOT MODIFY 3 3 */ 4 + import type * as SoSprkActorDefs from "./defs.ts"; 5 + 4 6 export type QueryParams = globalThis.Record<PropertyKey, never>; 5 7 export type InputSchema = undefined; 6 8 7 9 export interface OutputSchema { 8 - followMode?: 9 - | "bsky" 10 - | "sprk" 11 - | (string & globalThis.Record<PropertyKey, never>); 10 + preferences: SoSprkActorDefs.Preferences; 12 11 } 13 12 14 13 export type HandlerInput = void;

+3 -4

lex/types/so/sprk/actor/putPreferences.ts

··· 1 1 /** 2 2 * GENERATED CODE - DO NOT MODIFY 3 3 */ 4 + import type * as SoSprkActorDefs from "./defs.ts"; 5 + 4 6 export type QueryParams = globalThis.Record<PropertyKey, never>; 5 7 6 8 export interface InputSchema { 7 - followMode?: 8 - | "bsky" 9 - | "sprk" 10 - | (string & globalThis.Record<PropertyKey, never>); 9 + preferences: SoSprkActorDefs.Preferences; 11 10 } 12 11 13 12 export interface HandlerInput {

-8

lex/types/so/sprk/feed/defs.ts

··· 49 49 replyCount?: number; 50 50 repostCount?: number; 51 51 likeCount?: number; 52 - lookCount?: number; 53 52 indexedAt: string; 54 53 viewer?: ViewerState; 55 54 labels?: (ComAtprotoLabelDefs.Label)[]; ··· 93 92 $type?: "so.sprk.feed.defs#viewerState"; 94 93 repost?: string; 95 94 like?: string; 96 - look?: string; 97 95 threadMuted?: boolean; 98 96 replyDisabled?: boolean; 99 97 embeddingDisabled?: boolean; ··· 312 310 descriptionFacets?: (SoSprkRichtextFacet.Main)[]; 313 311 avatar?: string; 314 312 likeCount?: number; 315 - lookCount?: number; 316 313 acceptsInteractions?: boolean; 317 314 labels?: (ComAtprotoLabelDefs.Label)[]; 318 315 viewer?: GeneratorViewerState; 319 - contentMode?: 320 - | "so.sprk.feed.defs#contentModeUnspecified" 321 - | "so.sprk.feed.defs#contentModeVideo" 322 - | (string & globalThis.Record<PropertyKey, never>); 323 316 indexedAt: string; 324 317 } 325 318 ··· 336 329 export interface GeneratorViewerState { 337 330 $type?: "so.sprk.feed.defs#generatorViewerState"; 338 331 like?: string; 339 - look?: string; 340 332 } 341 333 342 334 const hashGeneratorViewerState = "generatorViewerState";

-4

lex/types/so/sprk/feed/generator.ts

··· 21 21 /** Declaration that a feed accepts feedback interactions from a client through so.sprk.feed.sendInteractions */ 22 22 acceptsInteractions?: boolean; 23 23 labels?: $Typed<ComAtprotoLabelDefs.SelfLabels> | { $type: string }; 24 - contentMode?: 25 - | "so.sprk.feed.defs#contentModeUnspecified" 26 - | "so.sprk.feed.defs#contentModeVideo" 27 - | (string & globalThis.Record<PropertyKey, never>); 28 24 createdAt: string; 29 25 [k: string]: unknown; 30 26 }

-32

lex/types/so/sprk/feed/getActorLooks.ts

··· 1 - /** 2 - * GENERATED CODE - DO NOT MODIFY 3 - */ 4 - import type * as SoSprkFeedDefs from "./defs.ts"; 5 - 6 - export type QueryParams = { 7 - actor: string; 8 - limit: number; 9 - cursor?: string; 10 - }; 11 - export type InputSchema = undefined; 12 - 13 - export interface OutputSchema { 14 - cursor?: string; 15 - feed: (SoSprkFeedDefs.FeedViewPost)[]; 16 - } 17 - 18 - export type HandlerInput = void; 19 - 20 - export interface HandlerSuccess { 21 - encoding: "application/json"; 22 - body: OutputSchema; 23 - headers?: { [key: string]: string }; 24 - } 25 - 26 - export interface HandlerError { 27 - status: number; 28 - message?: string; 29 - error?: "BlockedActor" | "BlockedByActor"; 30 - } 31 - 32 - export type HandlerOutput = HandlerError | HandlerSuccess;

-58

lex/types/so/sprk/feed/getLooks.ts

··· 1 - /** 2 - * GENERATED CODE - DO NOT MODIFY 3 - */ 4 - import { validate as _validate } from "../../../../lexicons.ts"; 5 - import { is$typed as _is$typed } from "../../../../util.ts"; 6 - import type * as SoSprkActorDefs from "../actor/defs.ts"; 7 - 8 - const is$typed = _is$typed, validate = _validate; 9 - const id = "so.sprk.feed.getLooks"; 10 - 11 - export type QueryParams = { 12 - /** AT-URI of the subject (eg, a post record). */ 13 - uri: string; 14 - /** CID of the subject record (aka, specific version of record), to filter looks. */ 15 - cid?: string; 16 - limit: number; 17 - cursor?: string; 18 - }; 19 - export type InputSchema = undefined; 20 - 21 - export interface OutputSchema { 22 - uri: string; 23 - cid?: string; 24 - cursor?: string; 25 - looks: (Look)[]; 26 - } 27 - 28 - export type HandlerInput = void; 29 - 30 - export interface HandlerSuccess { 31 - encoding: "application/json"; 32 - body: OutputSchema; 33 - headers?: { [key: string]: string }; 34 - } 35 - 36 - export interface HandlerError { 37 - status: number; 38 - message?: string; 39 - } 40 - 41 - export type HandlerOutput = HandlerError | HandlerSuccess; 42 - 43 - export interface Look { 44 - $type?: "so.sprk.feed.getLooks#look"; 45 - indexedAt: string; 46 - createdAt: string; 47 - actor: SoSprkActorDefs.ProfileView; 48 - } 49 - 50 - const hashLook = "look"; 51 - 52 - export function isLook<V>(v: V) { 53 - return is$typed(v, id, hashLook); 54 - } 55 - 56 - export function validateLook<V>(v: V) { 57 - return validate<Look & V>(v, id, hashLook); 58 - }

-26

lex/types/so/sprk/feed/look.ts

··· 1 - /** 2 - * GENERATED CODE - DO NOT MODIFY 3 - */ 4 - import { validate as _validate } from "../../../../lexicons.ts"; 5 - import { is$typed as _is$typed } from "../../../../util.ts"; 6 - import type * as ComAtprotoRepoStrongRef from "../../../com/atproto/repo/strongRef.ts"; 7 - 8 - const is$typed = _is$typed, validate = _validate; 9 - const id = "so.sprk.feed.look"; 10 - 11 - export interface Record { 12 - $type: "so.sprk.feed.look"; 13 - subject: ComAtprotoRepoStrongRef.Main; 14 - createdAt: string; 15 - [k: string]: unknown; 16 - } 17 - 18 - const hashRecord = "main"; 19 - 20 - export function isRecord<V>(v: V) { 21 - return is$typed(v, id, hashRecord); 22 - } 23 - 24 - export function validateRecord<V>(v: V) { 25 - return validate<Record & V>(v, id, hashRecord, true); 26 - }

+1 -25

lexicons/so/sprk/actor/defs.json

··· 182 182 "#adultContentPref", 183 183 "#contentLabelPref", 184 184 "#savedFeedsPref", 185 - "#savedFeedsPrefV2", 186 185 "#personalDetailsPref", 187 186 "#feedViewPref", 188 187 "#threadViewPref", ··· 236 235 } 237 236 } 238 237 }, 239 - "savedFeedsPrefV2": { 238 + "savedFeedsPref": { 240 239 "type": "object", 241 240 "required": ["items"], 242 241 "properties": { ··· 246 245 "type": "ref", 247 246 "ref": "so.sprk.actor.defs#savedFeed" 248 247 } 249 - } 250 - } 251 - }, 252 - "savedFeedsPref": { 253 - "type": "object", 254 - "required": ["pinned", "saved"], 255 - "properties": { 256 - "pinned": { 257 - "type": "array", 258 - "items": { 259 - "type": "string", 260 - "format": "at-uri" 261 - } 262 - }, 263 - "saved": { 264 - "type": "array", 265 - "items": { 266 - "type": "string", 267 - "format": "at-uri" 268 - } 269 - }, 270 - "timelineIndex": { 271 - "type": "integer" 272 248 } 273 249 } 274 250 },

+4 -3

lexicons/so/sprk/actor/getPreferences.json

··· 13 13 "encoding": "application/json", 14 14 "schema": { 15 15 "type": "object", 16 + "required": ["preferences"], 16 17 "properties": { 17 - "followMode": { 18 - "type": "string", 19 - "knownValues": ["bsky", "sprk"] 18 + "preferences": { 19 + "type": "ref", 20 + "ref": "so.sprk.actor.defs#preferences" 20 21 } 21 22 } 22 23 }

+4 -3

lexicons/so/sprk/actor/putPreferences.json

··· 9 9 "encoding": "application/json", 10 10 "schema": { 11 11 "type": "object", 12 + "required": ["preferences"], 12 13 "properties": { 13 - "followMode": { 14 - "type": "string", 15 - "knownValues": ["bsky", "sprk"] 14 + "preferences": { 15 + "type": "ref", 16 + "ref": "so.sprk.actor.defs#preferences" 16 17 } 17 18 } 18 19 }

+1 -12

lexicons/so/sprk/feed/defs.json

··· 39 39 "replyCount": { "type": "integer" }, 40 40 "repostCount": { "type": "integer" }, 41 41 "likeCount": { "type": "integer" }, 42 - "lookCount": { "type": "integer" }, 43 42 "indexedAt": { "type": "string", "format": "datetime" }, 44 43 "viewer": { "type": "ref", "ref": "#viewerState" }, 45 44 "labels": { ··· 75 74 "properties": { 76 75 "repost": { "type": "string", "format": "at-uri" }, 77 76 "like": { "type": "string", "format": "at-uri" }, 78 - "look": { "type": "string", "format": "at-uri" }, 79 77 "threadMuted": { "type": "boolean" }, 80 78 "replyDisabled": { "type": "boolean" }, 81 79 "embeddingDisabled": { "type": "boolean" }, ··· 219 217 }, 220 218 "avatar": { "type": "string", "format": "uri" }, 221 219 "likeCount": { "type": "integer", "minimum": 0 }, 222 - "lookCount": { "type": "integer", "minimum": 0 }, 223 220 "acceptsInteractions": { "type": "boolean" }, 224 221 "labels": { 225 222 "type": "array", 226 223 "items": { "type": "ref", "ref": "com.atproto.label.defs#label" } 227 224 }, 228 225 "viewer": { "type": "ref", "ref": "#generatorViewerState" }, 229 - "contentMode": { 230 - "type": "string", 231 - "knownValues": [ 232 - "so.sprk.feed.defs#contentModeUnspecified", 233 - "so.sprk.feed.defs#contentModeVideo" 234 - ] 235 - }, 236 226 "indexedAt": { "type": "string", "format": "datetime" } 237 227 } 238 228 }, 239 229 "generatorViewerState": { 240 230 "type": "object", 241 231 "properties": { 242 - "like": { "type": "string", "format": "at-uri" }, 243 - "look": { "type": "string", "format": "at-uri" } 232 + "like": { "type": "string", "format": "at-uri" } 244 233 } 245 234 }, 246 235 "skeletonFeedPost": {

-7

lexicons/so/sprk/feed/generator.json

··· 39 39 "description": "Self-label values", 40 40 "refs": ["com.atproto.label.defs#selfLabels"] 41 41 }, 42 - "contentMode": { 43 - "type": "string", 44 - "knownValues": [ 45 - "so.sprk.feed.defs#contentModeUnspecified", 46 - "so.sprk.feed.defs#contentModeVideo" 47 - ] 48 - }, 49 42 "createdAt": { "type": "string", "format": "datetime" } 50 43 } 51 44 }

-42

lexicons/so/sprk/feed/getActorLooks.json

··· 1 - { 2 - "lexicon": 1, 3 - "id": "so.sprk.feed.getActorLooks", 4 - "defs": { 5 - "main": { 6 - "type": "query", 7 - "description": "Get a list of posts looked by an actor. Requires auth, actor must be the requesting account.", 8 - "parameters": { 9 - "type": "params", 10 - "required": ["actor"], 11 - "properties": { 12 - "actor": { "type": "string", "format": "at-identifier" }, 13 - "limit": { 14 - "type": "integer", 15 - "minimum": 1, 16 - "maximum": 100, 17 - "default": 50 18 - }, 19 - "cursor": { "type": "string" } 20 - } 21 - }, 22 - "output": { 23 - "encoding": "application/json", 24 - "schema": { 25 - "type": "object", 26 - "required": ["feed"], 27 - "properties": { 28 - "cursor": { "type": "string" }, 29 - "feed": { 30 - "type": "array", 31 - "items": { 32 - "type": "ref", 33 - "ref": "so.sprk.feed.defs#feedViewPost" 34 - } 35 - } 36 - } 37 - } 38 - }, 39 - "errors": [{ "name": "BlockedActor" }, { "name": "BlockedByActor" }] 40 - } 41 - } 42 - }

-58

lexicons/so/sprk/feed/getLooks.json

··· 1 - { 2 - "lexicon": 1, 3 - "id": "so.sprk.feed.getLooks", 4 - "defs": { 5 - "main": { 6 - "type": "query", 7 - "description": "Get look records which reference a subject (by AT-URI and CID).", 8 - "parameters": { 9 - "type": "params", 10 - "required": ["uri"], 11 - "properties": { 12 - "uri": { 13 - "type": "string", 14 - "format": "at-uri", 15 - "description": "AT-URI of the subject (eg, a post record)." 16 - }, 17 - "cid": { 18 - "type": "string", 19 - "format": "cid", 20 - "description": "CID of the subject record (aka, specific version of record), to filter looks." 21 - }, 22 - "limit": { 23 - "type": "integer", 24 - "minimum": 1, 25 - "maximum": 100, 26 - "default": 50 27 - }, 28 - "cursor": { "type": "string" } 29 - } 30 - }, 31 - "output": { 32 - "encoding": "application/json", 33 - "schema": { 34 - "type": "object", 35 - "required": ["uri", "looks"], 36 - "properties": { 37 - "uri": { "type": "string", "format": "at-uri" }, 38 - "cid": { "type": "string", "format": "cid" }, 39 - "cursor": { "type": "string" }, 40 - "looks": { 41 - "type": "array", 42 - "items": { "type": "ref", "ref": "#look" } 43 - } 44 - } 45 - } 46 - } 47 - }, 48 - "look": { 49 - "type": "object", 50 - "required": ["indexedAt", "createdAt", "actor"], 51 - "properties": { 52 - "indexedAt": { "type": "string", "format": "datetime" }, 53 - "createdAt": { "type": "string", "format": "datetime" }, 54 - "actor": { "type": "ref", "ref": "so.sprk.actor.defs#profileView" } 55 - } 56 - } 57 - } 58 - }

-19

lexicons/so/sprk/feed/look.json

··· 1 - { 2 - "lexicon": 1, 3 - "id": "so.sprk.feed.look", 4 - "defs": { 5 - "main": { 6 - "type": "record", 7 - "description": "Record declaring a 'look' of a piece of subject content. Equivalent to a 'view'", 8 - "key": "tid", 9 - "record": { 10 - "type": "object", 11 - "required": ["subject", "createdAt"], 12 - "properties": { 13 - "subject": { "type": "ref", "ref": "com.atproto.repo.strongRef" }, 14 - "createdAt": { "type": "string", "format": "datetime" } 15 - } 16 - } 17 - } 18 - } 19 - }

+87 -1

services/takedown.ts

··· 13 13 targetCid: string; 14 14 reason: string; 15 15 adminDid: string; 16 + ref?: string; 16 17 }): Promise<void> { 17 - const { targetUri, targetCid, reason, adminDid } = params; 18 + const { targetUri, targetCid, reason, adminDid, ref } = params; 18 19 19 20 // Create a takedown record 20 21 await this.db.models.Takedown.create({ ··· 23 24 reason, 24 25 takenDownBy: adminDid, 25 26 takenDownAt: new Date().toISOString(), 27 + ref: ref || null, 26 28 applied: true, 27 29 }); 30 + 31 + // Update the record document with takedown status 32 + await this.updateRecordTakedownStatus( 33 + targetUri, 34 + true, 35 + ref || "BSKY-TAKEDOWN-UNKNOWN", 36 + ); 28 37 } 29 38 30 39 // Add a method to handle user repo takedowns ··· 45 54 ref: ref || null, 46 55 applied: false, 47 56 }); 57 + 58 + // Update all records from this DID with takedown status 59 + await this.updateAllRecordsTakedownStatusByDid( 60 + did, 61 + true, 62 + ref || "BSKY-TAKEDOWN-UNKNOWN", 63 + ); 48 64 } 49 65 50 66 // Add a method to handle blob takedowns ··· 108 124 109 125 async removeTakedown(targetUri: string): Promise<boolean> { 110 126 const result = await this.db.models.Takedown.deleteOne({ targetUri }); 127 + 128 + // Update the record document to remove takedown status 129 + if (result.deletedCount > 0) { 130 + await this.updateRecordTakedownStatus(targetUri, false); 131 + } 132 + 111 133 return result.deletedCount > 0; 112 134 } 113 135 114 136 // Add a method to remove repo takedown 115 137 async removeRepoTakedown(did: string): Promise<boolean> { 116 138 const result = await this.db.models.RepoTakedown.deleteOne({ did }); 139 + 140 + // Update all records from this DID to remove takedown status 141 + if (result.deletedCount > 0) { 142 + await this.updateAllRecordsTakedownStatusByDid(did, false); 143 + } 144 + 117 145 return result.deletedCount > 0; 118 146 } 119 147 ··· 284 312 const takedown = await this.db.models.BlobTakedown.findOne({ did, cid }) 285 313 .lean(); 286 314 return takedown; 315 + } 316 + 317 + /** 318 + * Update the takenDown and takedownRef properties on a RecordDocument 319 + * @param uri The URI of the record to update 320 + * @param takenDown Whether the record is taken down 321 + * @param takedownRef Optional reference for the takedown 322 + */ 323 + async updateRecordTakedownStatus( 324 + uri: string, 325 + takenDown: boolean, 326 + takedownRef?: string, 327 + ): Promise<void> { 328 + const updateData: { takenDown: boolean; takedownRef?: string } = { 329 + takenDown, 330 + }; 331 + 332 + if (takenDown && takedownRef) { 333 + updateData.takedownRef = takedownRef; 334 + await this.db.models.Record.updateOne( 335 + { uri }, 336 + { $set: updateData }, 337 + ); 338 + } else if (!takenDown) { 339 + await this.db.models.Record.updateOne( 340 + { uri }, 341 + { $set: { takenDown }, $unset: { takedownRef: "" } }, 342 + ); 343 + } 344 + } 345 + 346 + /** 347 + * Update the takenDown and takedownRef properties on all RecordDocuments for a specific DID 348 + * @param did The DID of the user whose records should be updated 349 + * @param takenDown Whether the records are taken down 350 + * @param takedownRef Optional reference for the takedown 351 + */ 352 + async updateAllRecordsTakedownStatusByDid( 353 + did: string, 354 + takenDown: boolean, 355 + takedownRef?: string, 356 + ): Promise<void> { 357 + if (takenDown && takedownRef) { 358 + await this.db.models.Record.updateMany( 359 + { did }, 360 + { $set: { takenDown, takedownRef } }, 361 + ); 362 + } else if (!takenDown) { 363 + await this.db.models.Record.updateMany( 364 + { did }, 365 + { $set: { takenDown }, $unset: { takedownRef: "" } }, 366 + ); 367 + } else { 368 + await this.db.models.Record.updateMany( 369 + { did }, 370 + { $set: { takenDown } }, 371 + ); 372 + } 287 373 } 288 374 }

+9 -25

utils/profile-helper.ts

··· 260 260 261 261 const now = new Date().toISOString(); 262 262 263 - // Get viewer preferences once for all profiles if viewer is authenticated 264 - let viewerFollowMode = "sprk"; 265 - 266 - if (viewerDid) { 267 - const viewerPref = await ctx.db.models.UserPreference.findOne({ 268 - userDid: viewerDid, 269 - }); 270 - viewerFollowMode = viewerPref?.followMode || "sprk"; 271 - } 272 - 273 263 // Helper function to get a single profile data 274 264 const getProfileData = async ( 275 265 actorParam: string, ··· 348 338 const handle = finalActorDoc.handle || 349 339 (await ctx.resolver.resolveDidToHandle(actorDid)); 350 340 351 - const actorPref = await ctx.db.models.UserPreference.findOne({ 352 - userDid: actorDid, 353 - }); 354 - const actorFollowMode = actorPref?.followMode || "sprk"; 355 - 356 341 // Twenty-four hours ago for recent stories 357 342 const twentyFourHoursAgo = new Date(); 358 343 twentyFourHoursAgo.setHours(twentyFourHoursAgo.getHours() - 24); ··· 387 372 // Count followers based on actor's follow mode preference 388 373 ctx.db.models.Follow.countDocuments({ 389 374 subject: actorDid, 390 - type: actorFollowMode, 391 375 }), 392 376 393 377 // Count follows based on actor's follow mode preference 394 378 ctx.db.models.Follow.countDocuments({ 395 379 authorDid: actorDid, 396 - type: actorFollowMode, 397 380 }), 398 381 399 382 // Count posts ··· 402 385 reply: null, 403 386 }), 404 387 405 - // Check for feed generators 388 + // Check for feed generators (bsky + sprk combined) 406 389 (async () => { 407 390 try { 408 - if (ctx.db.models.Generator) { 409 - return await ctx.db.models.Generator.countDocuments({ 391 + const [bskyCount, sprkCount] = await Promise.all([ 392 + ctx.db.models.BskyGenerator.countDocuments({ 393 + authorDid: actorDid, 394 + }), 395 + ctx.db.models.SprkGenerator.countDocuments({ 410 396 authorDid: actorDid, 411 - }); 412 - } 413 - return 0; 397 + }), 398 + ]); 399 + return bskyCount + sprkCount; 414 400 } catch (_error) { 415 401 return 0; 416 402 } ··· 421 407 ? ctx.db.models.Follow.findOne({ 422 408 subject: actorDid, 423 409 authorDid: viewerDid, 424 - type: viewerFollowMode, 425 410 }) 426 411 : Promise.resolve(null), 427 412 ··· 429 414 ? ctx.db.models.Follow.findOne({ 430 415 subject: viewerDid, 431 416 authorDid: actorDid, 432 - type: actorFollowMode, 433 417 }) 434 418 : Promise.resolve(null), 435 419

Configure Feed

Configure Feed