sprk.so / server
[READ ONLY MIRROR] Spark Social AppView Server github.com/sprksocial/server
atproto deno hono lexicon
1
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat: return cursor save status and add IdResolver cache

Roscoe Rubin-Rottenberg 367b16a9 c128b292

+53 -39
+1 -1
compose.dev.yaml
··· 70 70 build: 71 71 context: . 72 72 dockerfile: Dockerfile 73 - command: ["deno", "run", "-A", "--watch", "ingest/index.ts"] 73 + command: ["deno", "run", "-A", "--watch", "ingest.ts"] 74 74 environment: 75 75 NODE_ENV: development 76 76 HOST: 0.0.0.0
+4 -1
data-plane/db/index.ts
··· 15 15 16 16 constructor(private cfg: ServerConfig) { 17 17 this.idResolver = new IdResolver({ 18 + plcUrl: this.cfg.plcUrl, 18 19 didCache: new MemoryCache(HOUR, DAY), 19 20 }); 20 21 } ··· 241 242 async saveCursorState( 242 243 cursorPosition: number, 243 244 identifier = "last_processed_cursor", 244 - ): Promise<void> { 245 + ): Promise<boolean> { 245 246 try { 246 247 await this.models.CursorState.findOneAndUpdate( 247 248 { identifier }, ··· 251 252 }, 252 253 { upsert: true }, 253 254 ); 255 + return true; 254 256 } catch (error) { 255 257 console.error( 256 258 "Failed to save cursor state", 257 259 { error, cursorPosition, identifier }, 258 260 ); 261 + return false; 259 262 } 260 263 } 261 264 }
+12 -11
data-plane/indexing/plugins/reply.ts
··· 273 273 const parentPost = await db.models.Post.findOne({ 274 274 uri: replyIdx.reply.reply?.parent.uri, 275 275 }); 276 - const [parentReply, parentCrosspostReply, nativeReplyCount] = await Promise.all([ 277 - db.models.Reply.findOne({ 278 - uri: replyIdx.reply.reply?.parent.uri, 279 - }), 280 - db.models.CrosspostReply.findOne({ 281 - uri: replyIdx.reply.reply?.parent.uri, 282 - }), 283 - db.models.Reply.countDocuments({ 284 - "reply.parent.uri": replyIdx.reply.reply.parent.uri, 285 - }), 286 - ]); 276 + const [parentReply, parentCrosspostReply, nativeReplyCount] = await Promise 277 + .all([ 278 + db.models.Reply.findOne({ 279 + uri: replyIdx.reply.reply?.parent.uri, 280 + }), 281 + db.models.CrosspostReply.findOne({ 282 + uri: replyIdx.reply.reply?.parent.uri, 283 + }), 284 + db.models.Reply.countDocuments({ 285 + "reply.parent.uri": replyIdx.reply.reply.parent.uri, 286 + }), 287 + ]); 287 288 const replyCount = nativeReplyCount; 288 289 289 290 if (parentPost) {
+4 -2
data-plane/subscription.ts
··· 156 156 startCursor, 157 157 setCursorInterval: 30000, // Save cursor every 30 seconds 158 158 setCursor: async (cursor: number) => { 159 - await db.saveCursorState(cursor); 160 - console.info("Cursor saved to database", { cursor }); 159 + const didSave = await db.saveCursorState(cursor); 160 + if (didSave) { 161 + console.info("Cursor saved to database", { cursor }); 162 + } 161 163 }, 162 164 }); 163 165 const firehose = new Firehose({
+3 -1
data-plane/util.ts
··· 46 46 // Get nested replies (depth > 1) 47 47 if (depth > 1) { 48 48 const processedUris = new Set(directChildren.map((r) => r.uri)); 49 - const toProcess = [...directChildren.map((r) => ({ uri: r.uri, depth: 1 }))]; 49 + const toProcess = [ 50 + ...directChildren.map((r) => ({ uri: r.uri, depth: 1 })), 51 + ]; 50 52 51 53 while (toProcess.length > 0) { 52 54 const current = toProcess.shift()!;
+4 -4
deno.json
··· 2 2 "tasks": { 3 3 "dev:db": "mongod --dbpath ./devdb", 4 4 "dev:api": "deno run -A --watch main.ts", 5 - "dev:ingest": "deno run -A --watch ingest/index.ts", 5 + "dev:ingest": "deno run -A --watch ingest.ts", 6 6 "cleanup:orphan-crosspost-reply-records": "deno run -A scripts/delete_orphan_crosspost_reply_records.ts", 7 7 "compact:record-collection": "deno run -A scripts/compact_record_collection.ts", 8 8 "reindex:record-collection": "deno run -A scripts/compact_record_collection.ts", ··· 20 20 "@atp/identity": "jsr:@atp/identity@^0.1.0-alpha.2", 21 21 "@atp/lexicon": "jsr:@atp/lexicon@^0.1.0-alpha.4", 22 22 "@atp/repo": "jsr:@atp/repo@^0.1.0-alpha.5", 23 - "@atp/sync": "jsr:@atp/sync@^0.1.0-alpha.9", 23 + "@atp/sync": "jsr:@atp/sync@^0.1.0-alpha.10", 24 24 "@atp/syntax": "jsr:@atp/syntax@^0.1.0-alpha.2", 25 25 "@atp/xrpc": "jsr:@atp/xrpc@^0.1.0-alpha.4", 26 26 "@atp/xrpc-server": "jsr:@atp/xrpc-server@^0.1.0-alpha.9", 27 27 "@std/assert": "jsr:@std/assert@^1.0.19", 28 28 "dotenv": "npm:dotenv@^17.3.1", 29 - "hono": "jsr:@hono/hono@^4.12.8", 29 + "hono": "jsr:@hono/hono@^4.12.9", 30 30 "@std/encoding": "jsr:@std/encoding@^1.0.10", 31 31 "@atproto/api": "npm:@atproto/api@^0.18.21", 32 32 "jose": "npm:jose@^6.2.2", 33 - "mongoose": "npm:mongoose@^9.3.1", 33 + "mongoose": "npm:mongoose@^9.3.3", 34 34 "multiformats": "npm:multiformats@^13.4.2", 35 35 "p-queue": "npm:p-queue@^9.1.0", 36 36 "mongodb-memory-server-core": "npm:mongodb-memory-server-core@^11.0.1",
+16 -16
deno.lock
··· 8 8 "jsr:@atp/identity@~0.1.0-alpha.2": "0.1.0-alpha.2", 9 9 "jsr:@atp/lexicon@~0.1.0-alpha.4": "0.1.0-alpha.4", 10 10 "jsr:@atp/repo@~0.1.0-alpha.5": "0.1.0-alpha.5", 11 - "jsr:@atp/sync@~0.1.0-alpha.9": "0.1.0-alpha.9", 11 + "jsr:@atp/sync@~0.1.0-alpha.10": "0.1.0-alpha.10", 12 12 "jsr:@atp/syntax@~0.1.0-alpha.2": "0.1.0-alpha.2", 13 13 "jsr:@atp/xrpc-server@~0.1.0-alpha.9": "0.1.0-alpha.9", 14 14 "jsr:@atp/xrpc@~0.1.0-alpha.4": "0.1.0-alpha.4", 15 - "jsr:@hono/hono@^4.10.8": "4.12.8", 16 - "jsr:@hono/hono@^4.12.8": "4.12.8", 15 + "jsr:@hono/hono@^4.10.8": "4.12.9", 16 + "jsr:@hono/hono@^4.12.9": "4.12.9", 17 17 "jsr:@logtape/file@^1.2.2": "1.3.7", 18 18 "jsr:@logtape/logtape@^1.2.2": "1.3.7", 19 19 "jsr:@logtape/logtape@^1.3.7": "1.3.7", ··· 33 33 "npm:@atproto/sync@*": "0.1.39", 34 34 "npm:@bufbuild/protobuf@1.5.0": "1.5.0", 35 35 "npm:@ipld/dag-cbor@^9.2.5": "9.2.5", 36 - "npm:@opentelemetry/api@^1.9.0": "1.9.0", 36 + "npm:@opentelemetry/api@^1.9.0": "1.9.1", 37 37 "npm:dotenv@^17.3.1": "17.3.1", 38 38 "npm:jose@^6.2.2": "6.2.2", 39 39 "npm:lodash@*": "4.17.21", 40 40 "npm:mongodb-memory-server-core@^11.0.1": "11.0.1", 41 - "npm:mongoose@^9.3.1": "9.3.1", 41 + "npm:mongoose@^9.3.3": "9.3.3", 42 42 "npm:multiformats@^13.4.1": "13.4.2", 43 43 "npm:multiformats@^13.4.2": "13.4.2", 44 44 "npm:p-queue@^8.1.1": "8.1.1", ··· 105 105 "npm:multiformats@^13.4.1" 106 106 ] 107 107 }, 108 - "@atp/sync@0.1.0-alpha.9": { 109 - "integrity": "7a2c84f69bafc80cf705db7921a7aa12a659d4f82e187b5403be7bf426d28536", 108 + "@atp/sync@0.1.0-alpha.10": { 109 + "integrity": "1610fedb8babaefc84c114b5c07e691202b71917f54aa5710f4e1abea388af4d", 110 110 "dependencies": [ 111 111 "jsr:@atp/common@~0.1.0-alpha.9", 112 112 "jsr:@atp/identity", ··· 146 146 "@hono/hono@4.9.8": { 147 147 "integrity": "908150f13e90181a051a3af3bf15203aff00190682afedfd38824d0cb9299a95" 148 148 }, 149 - "@hono/hono@4.12.8": { 150 - "integrity": "1997fde8abf28e84d821c9c867229be020e268d4749397aec071305bf54db297" 149 + "@hono/hono@4.12.9": { 150 + "integrity": "53c2b0a721c1d782e6a347ad5bf15a3235ff624b98859c21f779b7ee2f5e040e" 151 151 }, 152 152 "@logtape/file@1.3.7": { 153 153 "integrity": "8cacd752ac49671135e80abc8cf4a843c377ab80906d075ece0e9105fc24677e", ··· 397 397 "@noble/hashes@1.8.0": { 398 398 "integrity": "sha512-jCs9ldd7NwzpgXDIf6P3+NrHh9/sD6CQdxHyjQI+h/6rDNo88ypBxxz45UDuZHz9r3tNz7N/VInSVoVdtXEI4A==" 399 399 }, 400 - "@opentelemetry/api@1.9.0": { 401 - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==" 400 + "@opentelemetry/api@1.9.1": { 401 + "integrity": "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==" 402 402 }, 403 403 "@types/webidl-conversions@7.0.3": { 404 404 "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==" ··· 827 827 "mongodb-connection-string-url" 828 828 ] 829 829 }, 830 - "mongoose@9.3.1": { 831 - "integrity": "sha512-58DuQti+LlRS74/UfWN4F3wZsC0Yr1dgTWZ2Wd3/TuSvm6rIdyAjDWbx2xGyuBooqJYdAWotVv4mQgVdivh+3Q==", 830 + "mongoose@9.3.3": { 831 + "integrity": "sha512-sfv5LOIPWeN5o/281kp4Rx9ZnuXb0g8CtvBTi7trYQs2PYYx8LWXegXxG3ar7VEns1o+d4h9LI/Dtc7dTTyYmA==", 832 832 "dependencies": [ 833 833 "kareem", 834 834 "mongodb", ··· 1246 1246 "jsr:@atp/identity@~0.1.0-alpha.2", 1247 1247 "jsr:@atp/lexicon@~0.1.0-alpha.4", 1248 1248 "jsr:@atp/repo@~0.1.0-alpha.5", 1249 - "jsr:@atp/sync@~0.1.0-alpha.9", 1249 + "jsr:@atp/sync@~0.1.0-alpha.10", 1250 1250 "jsr:@atp/syntax@~0.1.0-alpha.2", 1251 1251 "jsr:@atp/xrpc-server@~0.1.0-alpha.9", 1252 1252 "jsr:@atp/xrpc@~0.1.0-alpha.4", 1253 - "jsr:@hono/hono@^4.12.8", 1253 + "jsr:@hono/hono@^4.12.9", 1254 1254 "jsr:@std/assert@^1.0.19", 1255 1255 "jsr:@std/encoding@^1.0.10", 1256 1256 "npm:@atproto/api@~0.18.21", 1257 1257 "npm:dotenv@^17.3.1", 1258 1258 "npm:jose@^6.2.2", 1259 1259 "npm:mongodb-memory-server-core@^11.0.1", 1260 - "npm:mongoose@^9.3.1", 1260 + "npm:mongoose@^9.3.3", 1261 1261 "npm:multiformats@^13.4.2", 1262 1262 "npm:p-queue@^9.1.0", 1263 1263 "npm:structured-headers@^2.0.2"
+5 -2
ingest.ts
··· 1 1 import { RepoSubscription } from "./data-plane/subscription.ts"; 2 - import { IdResolver } from "@atp/identity"; 2 + import { IdResolver, MemoryCache } from "@atp/identity"; 3 3 import { ServerConfig } from "./config.ts"; 4 4 import { Database } from "./data-plane/db/index.ts"; 5 5 6 6 const cfg = ServerConfig.readEnv(); 7 7 8 - const idResolver = new IdResolver({ plcUrl: cfg.plcUrl }); 8 + const idResolver = new IdResolver({ 9 + plcUrl: cfg.plcUrl, 10 + didCache: new MemoryCache(), 11 + }); 9 12 const db = new Database(cfg); 10 13 db.connect(); 11 14
+4 -1
tests/util.ts
··· 213 213 ), 214 214 }; 215 215 216 + await connection.asPromise(); 217 + 216 218 // Seed data 217 219 await seedTestData(dbModels, opts); 218 220 ··· 273 275 ping: () => Promise.resolve(), 274 276 models: {}, 275 277 getCursorState: () => Promise.resolve(null), 276 - saveCursorState: () => Promise.resolve(), 278 + saveCursorState: () => Promise.resolve(true), 277 279 idResolver, 278 280 } as unknown as Database; 279 281 ··· 342 344 }, 343 345 { upsert: true }, 344 346 ); 347 + return true; 345 348 }, 346 349 resolveHandle: async (handle: string) => { 347 350 return await idResolver.handle.resolve(handle);