Spark feed generator template
ingester
+325
-6
+1
-2
README.md
+1
-2
README.md
+1
deno.json
+1
deno.json
···
7
7
"@atp/common": "jsr:@atp/common@^0.1.0-alpha.4",
8
8
"@atp/identity": "jsr:@atp/identity@^0.1.0-alpha.1",
9
9
"@atp/lexicon": "jsr:@atp/lexicon@^0.1.0-alpha.2",
10
+
"@atp/sync": "jsr:@atp/sync@^0.1.0-alpha.4",
10
11
"@atp/syntax": "jsr:@atp/syntax@^0.1.0-alpha.2",
11
12
"@atp/xrpc-server": "jsr:@atp/xrpc-server@^0.1.0-alpha.3",
12
13
"@logtape/logtape": "jsr:@logtape/logtape@^1.2.2",
+44
deno.lock
+44
deno.lock
···
9
9
"jsr:@atp/identity@~0.1.0-alpha.1": "0.1.0-alpha.1",
10
10
"jsr:@atp/lexicon@~0.1.0-alpha.1": "0.1.0-alpha.2",
11
11
"jsr:@atp/lexicon@~0.1.0-alpha.2": "0.1.0-alpha.2",
12
+
"jsr:@atp/repo@~0.1.0-alpha.2": "0.1.0-alpha.2",
13
+
"jsr:@atp/sync@~0.1.0-alpha.4": "0.1.0-alpha.4",
12
14
"jsr:@atp/syntax@~0.1.0-alpha.1": "0.1.0-alpha.2",
13
15
"jsr:@atp/syntax@~0.1.0-alpha.2": "0.1.0-alpha.2",
16
+
"jsr:@atp/xrpc-server@~0.1.0-alpha.2": "0.1.0-alpha.3",
14
17
"jsr:@atp/xrpc-server@~0.1.0-alpha.3": "0.1.0-alpha.3",
15
18
"jsr:@atp/xrpc@~0.1.0-alpha.2": "0.1.0-alpha.2",
16
19
"jsr:@hono/hono@^4.10.7": "4.10.7",
···
39
42
"npm:jose@^6.1.3": "6.1.3",
40
43
"npm:mongoose@^8.20.1": "8.20.1",
41
44
"npm:multiformats@^13.4.1": "13.4.1",
45
+
"npm:p-queue@^8.1.1": "8.1.1",
42
46
"npm:rate-limiter-flexible@^2.4.2": "2.4.2",
43
47
"npm:zod@^4.1.11": "4.1.13"
44
48
},
···
88
92
"npm:zod"
89
93
]
90
94
},
95
+
"@atp/repo@0.1.0-alpha.2": {
96
+
"integrity": "6da50453bbd527a679237d15bc9569eb2195503189f9be9d3023060f3f89f44a",
97
+
"dependencies": [
98
+
"jsr:@atp/bytes",
99
+
"jsr:@atp/common@~0.1.0-alpha.4",
100
+
"jsr:@atp/crypto@~0.1.0-alpha.2",
101
+
"jsr:@atp/lexicon@~0.1.0-alpha.2",
102
+
"jsr:@std/encoding",
103
+
"npm:@ipld/dag-cbor",
104
+
"npm:multiformats",
105
+
"npm:zod"
106
+
]
107
+
},
108
+
"@atp/sync@0.1.0-alpha.4": {
109
+
"integrity": "9b6aa6ccc9447270843272e40bfcb26520eddaf37f98202bcbab6c0bee0a602b",
110
+
"dependencies": [
111
+
"jsr:@atp/common@~0.1.0-alpha.4",
112
+
"jsr:@atp/identity",
113
+
"jsr:@atp/lexicon@~0.1.0-alpha.2",
114
+
"jsr:@atp/repo",
115
+
"jsr:@atp/syntax@~0.1.0-alpha.1",
116
+
"jsr:@atp/xrpc-server@~0.1.0-alpha.2",
117
+
"npm:multiformats",
118
+
"npm:p-queue"
119
+
]
120
+
},
91
121
"@atp/syntax@0.1.0-alpha.1": {
92
122
"integrity": "9e2055cace77cf63a8c52a4a94c39492215e7135101db7bc2289ebad9bec1991"
93
123
},
···
232
262
"tslib"
233
263
]
234
264
},
265
+
"eventemitter3@5.0.1": {
266
+
"integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="
267
+
},
235
268
"jose@6.1.3": {
236
269
"integrity": "sha512-0TpaTfihd4QMNwrz/ob2Bp7X04yuxJkjRGi4aKmOqwhov54i6u79oCv7T+C7lo70MKH6BesI3vscD1yb/yzKXQ=="
237
270
},
···
283
316
"multiformats@13.4.1": {
284
317
"integrity": "sha512-VqO6OSvLrFVAYYjgsr8tyv62/rCQhPgsZUXLTqoFLSgdkgiUYKYeArbt1uWLlEpkjxQe+P0+sHlbPEte1Bi06Q=="
285
318
},
319
+
"p-queue@8.1.1": {
320
+
"integrity": "sha512-aNZ+VfjobsWryoiPnEApGGmf5WmNsCo9xu8dfaYamG5qaLP7ClhLN6NgsFe6SwJ2UbLEBK5dv9x8Mn5+RVhMWQ==",
321
+
"dependencies": [
322
+
"eventemitter3",
323
+
"p-timeout"
324
+
]
325
+
},
326
+
"p-timeout@6.1.4": {
327
+
"integrity": "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg=="
328
+
},
286
329
"punycode@2.3.1": {
287
330
"integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="
288
331
},
···
329
372
"jsr:@atp/common@~0.1.0-alpha.4",
330
373
"jsr:@atp/identity@~0.1.0-alpha.1",
331
374
"jsr:@atp/lexicon@~0.1.0-alpha.2",
375
+
"jsr:@atp/sync@~0.1.0-alpha.4",
332
376
"jsr:@atp/syntax@~0.1.0-alpha.2",
333
377
"jsr:@atp/xrpc-server@~0.1.0-alpha.3",
334
378
"jsr:@hono/hono@^4.10.7",
+43
ingester/handlers/follow.ts
+43
ingester/handlers/follow.ts
···
1
+
import { CollectionHandler } from "../types.ts";
2
+
import type { Record as FollowRecord } from "../../lex/types/so/sprk/graph/follow.ts";
3
+
4
+
/** Handler for so.sprk.graph.follow collection */
5
+
const followHandler: CollectionHandler = {
6
+
collection: "so.sprk.graph.follow",
7
+
8
+
handleInsert: async (ctx, evt) => {
9
+
const record = evt.record as FollowRecord;
10
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
11
+
12
+
try {
13
+
await ctx.db.models.Follow.findOneAndUpdate(
14
+
{ uri },
15
+
{
16
+
uri,
17
+
cid: evt.cid,
18
+
authorDid: evt.did,
19
+
subject: record.subject,
20
+
createdAt: record.createdAt,
21
+
indexedAt: new Date().toISOString(),
22
+
},
23
+
{ upsert: true, new: true },
24
+
);
25
+
ctx.logger.debug("Indexed follow", { uri });
26
+
} catch (err) {
27
+
ctx.logger.error("Failed to index follow", { uri, error: err });
28
+
}
29
+
},
30
+
31
+
handleDelete: async (ctx, evt) => {
32
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
33
+
34
+
try {
35
+
await ctx.db.models.Follow.deleteOne({ uri });
36
+
ctx.logger.debug("Deleted follow", { uri });
37
+
} catch (err) {
38
+
ctx.logger.error("Failed to delete follow", { uri, error: err });
39
+
}
40
+
},
41
+
};
42
+
43
+
export default followHandler;
+46
ingester/handlers/like.ts
+46
ingester/handlers/like.ts
···
1
+
import { CollectionHandler } from "../types.ts";
2
+
import type { Record as LikeRecord } from "../../lex/types/so/sprk/feed/like.ts";
3
+
4
+
/** Handler for so.sprk.feed.like collection */
5
+
const likeHandler: CollectionHandler = {
6
+
collection: "so.sprk.feed.like",
7
+
8
+
handleInsert: async (ctx, evt) => {
9
+
const record = evt.record as LikeRecord;
10
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
11
+
12
+
try {
13
+
await ctx.db.models.Like.findOneAndUpdate(
14
+
{ uri },
15
+
{
16
+
uri,
17
+
cid: evt.cid,
18
+
authorDid: evt.did,
19
+
subject: record.subject.uri,
20
+
subjectCid: record.subject.cid,
21
+
via: record.via?.uri ?? null,
22
+
viaCid: record.via?.cid ?? null,
23
+
createdAt: record.createdAt,
24
+
indexedAt: new Date().toISOString(),
25
+
},
26
+
{ upsert: true, new: true },
27
+
);
28
+
ctx.logger.debug("Indexed like", { uri });
29
+
} catch (err) {
30
+
ctx.logger.error("Failed to index like", { uri, error: err });
31
+
}
32
+
},
33
+
34
+
handleDelete: async (ctx, evt) => {
35
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
36
+
37
+
try {
38
+
await ctx.db.models.Like.deleteOne({ uri });
39
+
ctx.logger.debug("Deleted like", { uri });
40
+
} catch (err) {
41
+
ctx.logger.error("Failed to delete like", { uri, error: err });
42
+
}
43
+
},
44
+
};
45
+
46
+
export default likeHandler;
+50
ingester/handlers/post.ts
+50
ingester/handlers/post.ts
···
1
+
import { CollectionHandler } from "../types.ts";
2
+
import type { Record as PostRecord } from "../../lex/types/so/sprk/feed/post.ts";
3
+
4
+
/** Handler for so.sprk.feed.post collection */
5
+
const postHandler: CollectionHandler = {
6
+
collection: "so.sprk.feed.post",
7
+
8
+
handleInsert: async (ctx, evt) => {
9
+
const record = evt.record as PostRecord;
10
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
11
+
12
+
try {
13
+
await ctx.db.models.Post.findOneAndUpdate(
14
+
{ uri },
15
+
{
16
+
uri,
17
+
cid: evt.cid,
18
+
authorDid: evt.did,
19
+
caption: record.caption,
20
+
media: record.media,
21
+
sound: record.sound,
22
+
langs: record.langs ?? [],
23
+
labels: (record.labels && "values" in record.labels)
24
+
? record.labels.values
25
+
: [],
26
+
tags: record.tags ?? [],
27
+
createdAt: record.createdAt,
28
+
indexedAt: new Date().toISOString(),
29
+
},
30
+
{ upsert: true, new: true },
31
+
);
32
+
ctx.logger.debug("Indexed post", { uri });
33
+
} catch (err) {
34
+
ctx.logger.error("Failed to index post", { uri, error: err });
35
+
}
36
+
},
37
+
38
+
handleDelete: async (ctx, evt) => {
39
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
40
+
41
+
try {
42
+
await ctx.db.models.Post.deleteOne({ uri });
43
+
ctx.logger.debug("Deleted post", { uri });
44
+
} catch (err) {
45
+
ctx.logger.error("Failed to delete post", { uri, error: err });
46
+
}
47
+
},
48
+
};
49
+
50
+
export default postHandler;
+46
ingester/handlers/repost.ts
+46
ingester/handlers/repost.ts
···
1
+
import { CollectionHandler } from "../types.ts";
2
+
import type { Record as RepostRecord } from "../../lex/types/so/sprk/feed/repost.ts";
3
+
4
+
/** Handler for so.sprk.feed.repost collection */
5
+
const repostHandler: CollectionHandler = {
6
+
collection: "so.sprk.feed.repost",
7
+
8
+
handleInsert: async (ctx, evt) => {
9
+
const record = evt.record as RepostRecord;
10
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
11
+
12
+
try {
13
+
await ctx.db.models.Repost.findOneAndUpdate(
14
+
{ uri },
15
+
{
16
+
uri,
17
+
cid: evt.cid,
18
+
authorDid: evt.did,
19
+
subject: record.subject.uri,
20
+
subjectCid: record.subject.cid,
21
+
via: record.via?.uri ?? null,
22
+
viaCid: record.via?.cid ?? null,
23
+
createdAt: record.createdAt,
24
+
indexedAt: new Date().toISOString(),
25
+
},
26
+
{ upsert: true, new: true },
27
+
);
28
+
ctx.logger.debug("Indexed repost", { uri });
29
+
} catch (err) {
30
+
ctx.logger.error("Failed to index repost", { uri, error: err });
31
+
}
32
+
},
33
+
34
+
handleDelete: async (ctx, evt) => {
35
+
const uri = `at://${evt.did}/${evt.collection}/${evt.rkey}`;
36
+
37
+
try {
38
+
await ctx.db.models.Repost.deleteOne({ uri });
39
+
ctx.logger.debug("Deleted repost", { uri });
40
+
} catch (err) {
41
+
ctx.logger.error("Failed to delete repost", { uri, error: err });
42
+
}
43
+
},
44
+
};
45
+
46
+
export default repostHandler;
+66
ingester/index.ts
+66
ingester/index.ts
···
1
+
import { Firehose } from "@atp/sync";
2
+
import { IdResolver } from "@atp/identity";
3
+
import { getLogger, Logger } from "@logtape/logtape";
4
+
import { Database } from "../db/connection.ts";
5
+
import { CollectionHandler, HandlerContext } from "./types.ts";
6
+
7
+
// collections
8
+
import postHandler from "./handlers/post.ts";
9
+
// uncomment the following lines if you want to ingest likes, reposts, or follows
10
+
// import likeHandler from "./handlers/like.ts";
11
+
// import repostHandler from "./handlers/repost.ts"
12
+
// import followHandler from "./handlers/follow.ts"
13
+
14
+
export class Ingester {
15
+
idResolver: IdResolver;
16
+
logger: Logger;
17
+
firehose: Firehose;
18
+
db: Database;
19
+
handlers: Map<string, CollectionHandler>;
20
+
21
+
constructor(
22
+
db: Database,
23
+
handlers: CollectionHandler[] = [
24
+
postHandler,
25
+
// uncomment the following lines to ingest likes, reposts, or follows
26
+
// likeHandler
27
+
// repostHandler
28
+
// followHandler
29
+
],
30
+
idResolver?: IdResolver,
31
+
) {
32
+
this.logger = getLogger(["feedgen", "ingester"]);
33
+
this.db = db;
34
+
this.idResolver = idResolver ?? new IdResolver();
35
+
36
+
// Build handler map for O(1) lookup
37
+
this.handlers = new Map(handlers.map((h) => [h.collection, h]));
38
+
39
+
const ctx: HandlerContext = {
40
+
db: this.db,
41
+
logger: this.logger,
42
+
};
43
+
44
+
this.firehose = new Firehose({
45
+
idResolver: this.idResolver,
46
+
handleEvent: async (evt) => {
47
+
if (!("collection" in evt)) return;
48
+
49
+
const handler = this.handlers.get(evt.collection);
50
+
if (!handler) return;
51
+
52
+
if (evt.event === "create" && handler.handleInsert) {
53
+
await handler.handleInsert(ctx, evt);
54
+
} else if (evt.event === "update" && handler.handleInsert) {
55
+
await handler.handleInsert(ctx, evt);
56
+
} else if (evt.event === "delete" && handler.handleDelete) {
57
+
await handler.handleDelete(ctx, evt);
58
+
}
59
+
},
60
+
onError: (err) => {
61
+
this.logger.error("Firehose error", { error: err });
62
+
},
63
+
filterCollections: handlers.map((h) => h.collection),
64
+
});
65
+
}
66
+
}
+22
ingester/types.ts
+22
ingester/types.ts
···
1
+
import { Event } from "@atp/sync";
2
+
import { Logger } from "@logtape/logtape";
3
+
import { Database } from "../db/connection.ts";
4
+
5
+
/** Context passed to collection handlers */
6
+
export interface HandlerContext {
7
+
db: Database;
8
+
logger: Logger;
9
+
}
10
+
11
+
/** Handler for a specific collection's events */
12
+
export interface CollectionHandler {
13
+
collection: string;
14
+
handleInsert?: (
15
+
ctx: HandlerContext,
16
+
evt: Event & { event: "create" | "update" },
17
+
) => Promise<void>;
18
+
handleDelete?: (
19
+
ctx: HandlerContext,
20
+
evt: Event & { event: "delete" },
21
+
) => Promise<void>;
22
+
}
+5
-3
main.ts
+5
-3
main.ts
···
11
11
import getFeedSkeleton from "./api/getFeedSkeleton.ts";
12
12
import wellKnown from "./api/well-known.ts";
13
13
import health from "./api/health.ts";
14
+
import { Ingester } from "./ingester/index.ts";
14
15
15
16
await configure({
16
17
sinks: {
···
94
95
Deno.exit(1);
95
96
}
96
97
98
+
const ingester = new Ingester(db);
99
+
ingester.firehose.start();
100
+
97
101
const { SPRK_HOST, SPRK_PORT } = env;
98
102
Deno.serve({
99
103
hostname: SPRK_HOST,
100
104
port: SPRK_PORT,
101
105
onListen: (info) => {
102
-
logger.info(`Server listening on ${info.hostname}:${info.port}`);
106
+
logger.info(`Server listening on http://${info.hostname}:${info.port}`);
103
107
},
104
108
}, app.fetch);
105
109
···
125
129
if (import.meta.main) {
126
130
startServer();
127
131
}
128
-
129
-
export default app;
+1
-1
utils/env.ts
+1
-1
utils/env.ts
···
8
8
9
9
SPRK_FEEDGEN_DOMAIN: envStr("FEEDGEN_DOMAIN"),
10
10
SPRK_DB_NAME: envStr("SPRK_DB_NAME"),
11
-
SPRK_DB_URI: envStr("SPRK_DB_URI") ?? "mongo://localhost:27017",
11
+
SPRK_DB_URI: envStr("SPRK_DB_URI") ?? "mongodb://localhost:27017",
12
12
SPRK_DB_USER: envStr("SPRK_DB_USER"),
13
13
SPRK_DB_PASS: envStr("SPRK_DB_PASS"),
14
14