[READ ONLY MIRROR] Spark Social AppView Server
github.com/sprksocial/server
atproto
deno
hono
lexicon
HOUSEKEEPING (#44)
* housekeeping pt1
* fix docker compose and a lil more cleanup
* cfg
* dev without docker
* Update main.ts
* getFollows/getFollowers
* Update config.ts
* sm updates
authored by
Roscoe Rubin-Rottenberg
and committed by
GitHub
abade5fa
b63f3340
+1063
-1876
+4
-11
Dockerfile
+4
-11
Dockerfile
···
1
-
FROM denoland/deno:alpine-2.3.5 AS builder
2
-
ARG COMMIT_SHA
3
-
4
-
WORKDIR /app
5
-
6
-
COPY . .
7
-
8
-
RUN deno cache main.ts
9
-
10
-
FROM denoland/deno:alpine-2.3.5 AS production
1
+
FROM denoland/deno:alpine-2.5.4
11
2
ARG COMMIT_SHA
12
3
ENV COMMIT_SHA=$COMMIT_SHA
13
4
···
15
6
16
7
WORKDIR /app
17
8
18
-
COPY --from=builder /app .
9
+
COPY . .
10
+
11
+
RUN deno install
19
12
20
13
EXPOSE 3000
21
14
-11
Dockerfile.dev
-11
Dockerfile.dev
+1
-1
LICENSE
+1
-1
LICENSE
+6
-11
README.md
+6
-11
README.md
···
21
21
22
22
```
23
23
# Database
24
-
DB_HOST=localhost
25
-
DB_PORT=27017
26
-
DB_NAME=dev
27
-
DB_USER=mongo
28
-
DB_PASSWORD=mongo
24
+
SPRK_DB_URI=mongodb://mongo:mongo@localhost:27017
29
25
30
26
# Server
31
-
HOST=0.0.0.0
32
27
NODE_ENV=development
33
-
PORT=4000
34
-
PUBLIC_URL=http://localhost:3000
35
-
SERVICE_DID=did:web:localhost
28
+
SPRK_PORT=4000
29
+
SPRK_PUBLIC_URL=http://localhost:3000
30
+
SPRK_SERVER_DID=did:web:localhost
36
31
37
32
# Keys, generate these with openssl ecparam --name secp256k1 --genkey --noout --outform DER | tail --bytes=+8 | head --bytes=32 | xxd --plain --cols 32
38
33
# On Mac: openssl ecparam -name secp256k1 -genkey -noout -outform DER | tail --bytes=+8 | head --bytes=32 | xxd --plain --cols 32
39
-
APPVIEW_K256_PRIVATE_KEY_HEX=keyhex
40
-
ADMIN_PASSWORD=password
34
+
SPRK_PRIVATE_KEY=keyhex
35
+
SPRK_ADMIN_PASSWORDS=password1,password2
41
36
```
+14
-20
api/com/atproto/admin/getAccountInfos.ts
+14
-20
api/com/atproto/admin/getAccountInfos.ts
···
1
-
import { AppContext } from "../../../../main.ts";
2
1
import { mapDefined } from "@atp/common";
3
2
import { INVALID_HANDLE } from "@atp/syntax";
3
+
import { AppContext } from "../../../../context.ts";
4
4
import { Server } from "../../../../lex/index.ts";
5
-
import { AuthRequiredError } from "@atp/xrpc-server";
6
5
7
6
export default function (server: Server, ctx: AppContext) {
8
7
server.com.atproto.admin.getAccountInfos({
9
8
auth: ctx.authVerifier.optionalStandardOrRole,
10
9
handler: async ({ params, auth }) => {
11
10
const { dids } = params;
11
+
const { includeTakedowns } = ctx.authVerifier.parseCreds(auth);
12
12
13
-
const { includeTakedowns } = ctx.authVerifier.parseCreds(auth);
14
-
if (!includeTakedowns) {
15
-
throw new AuthRequiredError("Requires admin privileges");
16
-
}
13
+
const actors = await ctx.hydrator.actor.getActors(dids, {
14
+
includeTakedowns: true,
15
+
});
17
16
18
-
const infos = await Promise.all(mapDefined(dids, async (did) => {
19
-
const info = await ctx.db.models.Actor.findOne({ did });
17
+
const infos = mapDefined(dids, (did) => {
18
+
const info = actors.get(did);
20
19
if (!info) return;
21
-
const profileRecord = await ctx.db.models.Profile.findOne({
22
-
authorDid: did,
23
-
});
24
-
const profileObj = profileRecord
25
-
? { ...profileRecord.toJSON(), _id: undefined, __v: undefined }
20
+
if (info.takedownRef && !includeTakedowns) return;
21
+
const profileRecord = !info.profileTakedownRef || includeTakedowns
22
+
? info.profile
26
23
: undefined;
27
24
28
25
return {
29
-
$type: "com.atproto.admin.defs#accountView" as const,
30
26
did,
31
27
handle: info.handle ?? INVALID_HANDLE,
32
-
relatedRecords: profileObj ? [profileObj] : undefined,
33
-
indexedAt: info.indexedAt,
28
+
relatedRecords: profileRecord ? [profileRecord] : undefined,
29
+
indexedAt: (info.sortedAt ?? new Date(0)).toISOString(),
34
30
};
35
-
}));
31
+
});
36
32
37
33
return {
38
34
encoding: "application/json",
39
35
body: {
40
-
infos: infos.filter((info): info is NonNullable<typeof info> =>
41
-
info != null
42
-
),
36
+
infos,
43
37
},
44
38
};
45
39
},
+58
-74
api/com/atproto/admin/getSubjectStatus.ts
+58
-74
api/com/atproto/admin/getSubjectStatus.ts
···
1
-
import { AppContext } from "../../../../main.ts";
1
+
import { InvalidRequestError } from "@atp/xrpc-server";
2
+
import { AppContext } from "../../../../context.ts";
2
3
import { Server } from "../../../../lex/index.ts";
3
-
import { AuthRequiredError, XRPCError } from "@atp/xrpc-server";
4
+
import { OutputSchema } from "../../../../lex/types/com/atproto/admin/getSubjectStatus.ts";
4
5
5
6
export default function (server: Server, ctx: AppContext) {
6
7
server.com.atproto.admin.getSubjectStatus({
7
-
auth: ctx.authVerifier.optionalStandardOrRole,
8
-
handler: async ({ params, auth }) => {
8
+
auth: ctx.authVerifier.roleOrModService,
9
+
handler: async ({ params }) => {
9
10
const { did, uri, blob } = params;
10
11
11
-
const { includeTakedowns } = ctx.authVerifier.parseCreds(auth);
12
-
if (!includeTakedowns) {
13
-
throw new AuthRequiredError("Requires admin privileges");
14
-
}
15
-
16
-
let subject: { $type: string } & Record<string, string>;
17
-
let takedown: { applied: boolean; ref: string | undefined } | undefined;
18
-
19
-
if (did) {
20
-
const actor = await ctx.db.models.Actor.findOne({ did });
21
-
const repoTakedown = await ctx.db.models.RepoTakedown.findOne({
22
-
subjectDid: did,
23
-
});
24
-
if (!actor) {
25
-
throw new XRPCError(404, "NotFound", "Actor not found");
12
+
let body: OutputSchema | null = null;
13
+
if (blob) {
14
+
if (!did) {
15
+
throw new InvalidRequestError(
16
+
"Must provide a did to request blob state",
17
+
);
26
18
}
27
-
subject = {
28
-
$type: "com.atproto.admin.defs#repoRef",
29
-
did: actor.did,
19
+
const res = await ctx.dataplane.moderation.getBlobTakedown(
20
+
did,
21
+
blob,
22
+
);
23
+
body = {
24
+
subject: {
25
+
$type: "com.atproto.admin.defs#repoBlobRef",
26
+
did: did,
27
+
cid: blob,
28
+
},
29
+
takedown: {
30
+
applied: res.takenDown,
31
+
ref: res.takedownRef ? "TAKEDOWN" : undefined,
32
+
},
30
33
};
31
-
if (repoTakedown) {
32
-
takedown = {
33
-
applied: repoTakedown.applied,
34
-
ref: repoTakedown.ref || undefined,
35
-
};
36
-
}
37
34
} else if (uri) {
38
-
const record = (await ctx.db.models.Profile.findOne({ uri })) ??
39
-
(await ctx.db.models.Post.findOne({ uri })) ??
40
-
(await ctx.db.models.Audio.findOne({ uri }));
41
-
const recordTakedown = await ctx.db.models.Takedown.findOne({
42
-
subjectUri: uri,
43
-
});
44
-
if (!record) {
45
-
throw new XRPCError(404, "NotFound", "Record not found");
46
-
}
47
-
subject = {
48
-
$type: "com.atproto.admin.defs#recordRef",
49
-
uri: record.uri,
50
-
cid: record.cid,
51
-
};
52
-
if (recordTakedown) {
53
-
takedown = {
54
-
applied: recordTakedown.applied,
55
-
ref: recordTakedown.ref || undefined,
35
+
const res = await ctx.hydrator.getRecord(uri, true);
36
+
if (res) {
37
+
body = {
38
+
subject: {
39
+
$type: "com.atproto.repo.strongRef",
40
+
uri,
41
+
cid: res.cid,
42
+
},
43
+
takedown: {
44
+
applied: !!res.takedownRef,
45
+
ref: res.takedownRef || undefined,
46
+
},
56
47
};
57
48
}
58
-
} else if (blob) {
59
-
const blobRecord = (await ctx.db.models.Profile.findOne({ blob })) ??
60
-
(await ctx.db.models.Post.findOne({ blob })) ??
61
-
(await ctx.db.models.Audio.findOne({ blob }));
62
-
if (!blobRecord) {
63
-
throw new XRPCError(404, "NotFound", "Blob record not found");
64
-
}
65
-
subject = {
66
-
$type: "com.atproto.admin.defs#repoBlobRef",
67
-
did: blobRecord.authorDid,
68
-
cid: blobRecord.cid,
69
-
recordUri: blobRecord.uri,
70
-
};
71
-
const blobTakedown = await ctx.db.models.BlobTakedown.findOne({
72
-
subjectDid: blobRecord.authorDid,
73
-
subjectCid: blobRecord.cid,
74
-
});
75
-
if (blobTakedown) {
76
-
takedown = {
77
-
applied: blobTakedown.applied,
78
-
ref: blobTakedown.ref || undefined,
49
+
} else if (did) {
50
+
const res = (
51
+
await ctx.hydrator.actor.getActors([did], {
52
+
includeTakedowns: true,
53
+
})
54
+
).get(did);
55
+
if (res) {
56
+
body = {
57
+
subject: {
58
+
$type: "com.atproto.admin.defs#repoRef",
59
+
did: did,
60
+
},
61
+
takedown: {
62
+
applied: !!res.takedownRef,
63
+
ref: res.takedownRef || undefined,
64
+
},
79
65
};
80
66
}
81
67
} else {
82
-
throw new XRPCError(
83
-
400,
84
-
"InvalidRequest",
85
-
"Must provide did, uri, or blob",
86
-
);
68
+
throw new InvalidRequestError("No provided subject");
87
69
}
88
-
70
+
if (body === null) {
71
+
throw new InvalidRequestError("Subject not found", "NotFound");
72
+
}
89
73
return {
90
74
encoding: "application/json",
91
-
body: { subject, takedown },
75
+
body,
92
76
};
93
77
},
94
78
});
+44
-93
api/com/atproto/admin/updateSubjectStatus.ts
+44
-93
api/com/atproto/admin/updateSubjectStatus.ts
···
1
-
import { HTTPException } from "hono/http-exception";
2
-
import { AppContext } from "../../../../main.ts";
1
+
import { AuthRequiredError, InvalidRequestError } from "@atp/xrpc-server";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { Server } from "../../../../lex/index.ts";
4
-
import type * as ComAtprotoAdminDefs from "../../../../lex/types/com/atproto/admin/defs.ts";
5
-
import type * as ComAtprotoRepoStrongRef from "../../../../lex/types/com/atproto/repo/strongRef.ts";
6
-
import { AuthRequiredError } from "@atp/xrpc-server";
4
+
import {
5
+
isRepoBlobRef,
6
+
isRepoRef,
7
+
} from "../../../../lex/types/com/atproto/admin/defs.ts";
8
+
import { isMain as isStrongRef } from "../../../../lex/types/com/atproto/repo/strongRef.ts";
7
9
8
10
export default function (server: Server, ctx: AppContext) {
9
11
server.com.atproto.admin.updateSubjectStatus({
10
-
auth: ctx.authVerifier.optionalStandardOrRole,
12
+
auth: ctx.authVerifier.roleOrModService,
11
13
handler: async ({ input, auth }) => {
12
-
const { subject, takedown } = input.body;
13
-
if (!takedown || typeof takedown.applied !== "boolean") {
14
-
throw new HTTPException(400, { message: "Invalid takedown status" });
15
-
}
16
-
17
14
const { canPerformTakedown } = ctx.authVerifier.parseCreds(auth);
18
15
if (!canPerformTakedown) {
19
-
throw new AuthRequiredError("Requires admin privileges");
16
+
throw new AuthRequiredError(
17
+
"Must be a full moderator to update subject state",
18
+
);
20
19
}
21
-
22
-
try {
23
-
if (subject.$type === "com.atproto.admin.defs#repoRef") {
24
-
const repoRef = subject as ComAtprotoAdminDefs.RepoRef;
25
-
if (!repoRef.did) {
26
-
throw new HTTPException(400, {
27
-
message: "DID is required for repo takedowns",
28
-
});
29
-
}
30
-
20
+
const { subject, takedown } = input.body;
21
+
if (takedown) {
22
+
if (isRepoRef(subject)) {
31
23
if (takedown.applied) {
32
-
await ctx.takedownService.takedownRepo({
33
-
did: repoRef.did,
34
-
reason: "Moderation action",
35
-
adminDid: auth.credentials.type === "standard"
36
-
? auth.credentials.iss
37
-
: "admin",
38
-
ref: takedown.ref,
39
-
});
40
-
await ctx.takedownService.updateRepoTakedownApplied(
41
-
repoRef.did,
42
-
true,
24
+
await ctx.dataplane.moderation.takedownActor(
25
+
subject.did,
26
+
takedown.ref,
43
27
);
44
28
} else {
45
-
await ctx.takedownService.removeRepoTakedown(repoRef.did);
46
-
}
47
-
} else if (subject.$type === "com.atproto.admin.defs#recordRef") {
48
-
const recordRef = subject as ComAtprotoRepoStrongRef.Main;
49
-
if (!recordRef.uri || !recordRef.cid) {
50
-
throw new HTTPException(400, {
51
-
message: "URI and CID are required for record takedowns",
52
-
});
29
+
await ctx.dataplane.moderation.untakedownActor(
30
+
subject.did,
31
+
);
53
32
}
54
-
33
+
} else if (isStrongRef(subject)) {
55
34
if (takedown.applied) {
56
-
await ctx.takedownService.takedownContent({
57
-
targetUri: recordRef.uri,
58
-
targetCid: recordRef.cid,
59
-
reason: "Moderation action",
60
-
adminDid: auth.credentials.type === "standard"
61
-
? auth.credentials.iss
62
-
: "admin",
63
-
ref: takedown.ref,
64
-
});
65
-
await ctx.takedownService.updateTakedownApplied(
66
-
recordRef.uri,
67
-
true,
35
+
await ctx.dataplane.moderation.takedownRecord(
36
+
subject.uri,
37
+
takedown.ref,
68
38
);
69
39
} else {
70
-
await ctx.takedownService.removeTakedown(recordRef.uri);
71
-
}
72
-
} else if (subject.$type === "com.atproto.admin.defs#repoBlobRef") {
73
-
const repoBlobRef = subject as ComAtprotoAdminDefs.RepoBlobRef;
74
-
if (!repoBlobRef.did || !repoBlobRef.cid) {
75
-
throw new HTTPException(400, {
76
-
message: "DID and CID are required for blob takedowns",
77
-
});
40
+
await ctx.dataplane.moderation.untakedownRecord(
41
+
subject.uri,
42
+
);
78
43
}
79
-
44
+
} else if (isRepoBlobRef(subject)) {
80
45
if (takedown.applied) {
81
-
await ctx.takedownService.takedownBlob({
82
-
did: repoBlobRef.did,
83
-
cid: repoBlobRef.cid,
84
-
reason: "Moderation action",
85
-
adminDid: auth.credentials.type === "standard"
86
-
? auth.credentials.iss
87
-
: "admin",
88
-
ref: takedown.ref,
89
-
});
90
-
await ctx.takedownService.updateBlobTakedownApplied(
91
-
repoBlobRef.did,
92
-
repoBlobRef.cid,
93
-
true,
46
+
await ctx.dataplane.moderation.takedownBlob(
47
+
subject.did,
48
+
subject.cid,
49
+
takedown.ref,
94
50
);
95
51
} else {
96
-
await ctx.takedownService.removeBlobTakedown(
97
-
repoBlobRef.did,
98
-
repoBlobRef.cid,
52
+
await ctx.dataplane.moderation.untakedownBlob(
53
+
subject.did,
54
+
subject.cid,
99
55
);
100
56
}
101
57
} else {
102
-
throw new HTTPException(400, {
103
-
message: `Unsupported subject type: ${subject.$type}`,
104
-
});
58
+
throw new InvalidRequestError("Invalid subject");
105
59
}
106
-
107
-
return {
108
-
encoding: "application/json",
109
-
body: {
110
-
subject,
111
-
takedown: takedown.applied ? takedown : undefined,
112
-
},
113
-
};
114
-
} catch (err) {
115
-
if (err instanceof HTTPException) throw err;
116
-
throw new HTTPException(500, { message: "Internal server error" });
117
60
}
61
+
62
+
return {
63
+
encoding: "application/json",
64
+
body: {
65
+
subject,
66
+
takedown,
67
+
},
68
+
};
118
69
},
119
70
});
120
71
}
+1
-1
api/com/atproto/identity/resolveHandle.ts
+1
-1
api/com/atproto/identity/resolveHandle.ts
···
1
1
import * as ident from "@atp/syntax";
2
2
import { InvalidRequestError } from "@atp/xrpc-server";
3
3
import { Server } from "../../../../lex/index.ts";
4
-
import { AppContext } from "../../../../main.ts";
4
+
import { AppContext } from "../../../../context.ts";
5
5
6
6
export default function (server: Server, ctx: AppContext) {
7
7
server.com.atproto.identity.resolveHandle({
+1
-1
api/com/atproto/repo/getRecord.ts
+1
-1
api/com/atproto/repo/getRecord.ts
···
1
1
import { AtUri } from "@atp/syntax";
2
2
import { InvalidRequestError } from "@atp/xrpc-server";
3
-
import { AppContext } from "../../../../main.ts";
3
+
import { AppContext } from "../../../../context.ts";
4
4
import { Server } from "../../../../lex/index.ts";
5
5
6
6
export default function (server: Server, ctx: AppContext) {
+35
api/health.ts
+35
api/health.ts
···
1
+
import { Hono } from "hono";
2
+
3
+
const app = new Hono();
4
+
5
+
app.get("/", (c) => {
6
+
return c.text(
7
+
`
8
+
.------..------..------..------..------.
9
+
|S.--. ||P.--. ||A.--. ||R.--. ||K.--. |
10
+
| :/\\: || :/\\: || (\\/) || :(): || :/\\: |
11
+
| :\\/: || (__) || :\\/: || ()() || :\\/: |
12
+
| '--'S|| '--'P|| '--'A|| '--'R|| '--'K|
13
+
${"`"}------'${"`"}------'${"`"}------'${"`"}------'${"`"}------'
14
+
15
+
16
+
This is an AT Protocol Application View (AppView) for the "sprk.so" application.
17
+
18
+
Most API routes are under /xrpc/
19
+
20
+
`,
21
+
);
22
+
});
23
+
24
+
app.get("/robots.txt", (c) => {
25
+
return c.text(
26
+
'# Hello Friends!\n\n# Crawling the public parts of the API is allowed. HTTP 429 ("backoff") status codes are used for rate-limiting. Up to a handful concurrent requests should be ok.\nUser-agent: *\nAllow: /',
27
+
);
28
+
});
29
+
30
+
app.get("/xrpc/_health", (c) => {
31
+
const version = Deno.env.get("COMMIT_SHA") ?? "unknown";
32
+
return c.json({ version });
33
+
});
34
+
35
+
export default app;
+1
-1
api/index.ts
+1
-1
api/index.ts
···
1
1
import { Server } from "../lex/index.ts";
2
-
import { AppContext } from "../main.ts";
2
+
import { AppContext } from "../context.ts";
3
3
import getAccountInfos from "./com/atproto/admin/getAccountInfos.ts";
4
4
import getSubjectStatus from "./com/atproto/admin/getSubjectStatus.ts";
5
5
import updateSubjectStatus from "./com/atproto/admin/updateSubjectStatus.ts";
+1
-1
api/so/sprk/actor/getPreferences.ts
+1
-1
api/so/sprk/actor/getPreferences.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { Preferences } from "../../../../lex/types/so/sprk/actor/defs.ts";
4
4
5
5
export default function (server: Server, ctx: AppContext) {
+1
-1
api/so/sprk/actor/getProfile.ts
+1
-1
api/so/sprk/actor/getProfile.ts
+1
-1
api/so/sprk/actor/getProfiles.ts
+1
-1
api/so/sprk/actor/getProfiles.ts
+1
-1
api/so/sprk/actor/putPreferences.ts
+1
-1
api/so/sprk/actor/putPreferences.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
2
import { SavedFeedsPref } from "../../../../lex/types/so/sprk/actor/defs.ts";
3
-
import { AppContext } from "../../../../main.ts";
3
+
import { AppContext } from "../../../../context.ts";
4
4
5
5
export default function (server: Server, ctx: AppContext) {
6
6
server.so.sprk.actor.putPreferences({
+1
-1
api/so/sprk/actor/searchActors.ts
+1
-1
api/so/sprk/actor/searchActors.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import type * as SoSprkActorSearch from "../../../../lex/types/so/sprk/actor/searchActors.ts";
4
4
import { getProfileViews } from "../../../../utils/profile-helper.ts";
5
5
+1
-1
api/so/sprk/feed/getAuthorFeed.ts
+1
-1
api/so/sprk/feed/getAuthorFeed.ts
···
1
1
import { mapDefined } from "@atp/common";
2
2
import { InvalidRequestError } from "@atp/xrpc-server";
3
-
import { AppContext } from "../../../../main.ts";
3
+
import { AppContext } from "../../../../context.ts";
4
4
import { DataPlane } from "../../../../data-plane/index.ts";
5
5
import { Actor } from "../../../../hydration/actor.ts";
6
6
import { FeedItem, Post } from "../../../../hydration/feed.ts";
+1
-1
api/so/sprk/feed/getPostThread.ts
+1
-1
api/so/sprk/feed/getPostThread.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { OutputSchema } from "../../../../lex/types/so/sprk/feed/getPostThread.ts";
4
4
import type * as SoSprkFeedDefs from "../../../../lex/types/so/sprk/feed/defs.ts";
5
5
import { transformPostsToPostViews } from "../../../../utils/post-transformer.ts";
+1
-1
api/so/sprk/feed/getPosts.ts
+1
-1
api/so/sprk/feed/getPosts.ts
+1
-1
api/so/sprk/feed/getStories.ts
+1
-1
api/so/sprk/feed/getStories.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { OutputSchema } from "../../../../lex/types/so/sprk/feed/getStories.ts";
4
4
import { transformStoriesToStoryViews } from "../../../../utils/story-transformer.ts";
5
5
import { StoryDocument } from "../../../../data-plane/db/models.ts";
+1
-1
api/so/sprk/feed/getStoriesTimeline.ts
+1
-1
api/so/sprk/feed/getStoriesTimeline.ts
···
1
1
import { InvalidRequestError } from "@atp/xrpc-server";
2
2
import { Server } from "../../../../lex/index.ts";
3
-
import { AppContext } from "../../../../main.ts";
3
+
import { AppContext } from "../../../../context.ts";
4
4
import { transformStoriesToStoryViews } from "../../../../utils/story-transformer.ts";
5
5
import { decodeBase64, encodeBase64 } from "@std/encoding";
6
6
import type { ProfileViewBasic } from "../../../../lex/types/so/sprk/actor/defs.ts";
+1
-1
api/so/sprk/feed/getSuggestedFeeds.ts
+1
-1
api/so/sprk/feed/getSuggestedFeeds.ts
+1
-1
api/so/sprk/feed/getTimeline.ts
+1
-1
api/so/sprk/feed/getTimeline.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { transformPostsToPostViews } from "../../../../utils/post-transformer.ts";
4
4
import { decodeBase64, encodeBase64 } from "@std/encoding";
5
5
import { OutputSchema } from "../../../../lex/types/so/sprk/feed/getTimeline.ts";
+1
-1
api/so/sprk/feed/searchPosts.ts
+1
-1
api/so/sprk/feed/searchPosts.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { transformPostsToPostViews } from "../../../../utils/post-transformer.ts";
4
4
import * as SoSprkFeedDefs from "../../../../lex/types/so/sprk/feed/defs.ts";
5
5
import { OutputSchema } from "../../../../lex/types/so/sprk/feed/searchPosts.ts";
+129
-64
api/so/sprk/graph/getFollowers.ts
+129
-64
api/so/sprk/graph/getFollowers.ts
···
1
+
import { mapDefined } from "@atp/common";
2
+
import { InvalidRequestError } from "@atp/xrpc-server";
3
+
import { AppContext } from "../../../../context.ts";
4
+
import {
5
+
HydrateCtx,
6
+
Hydrator,
7
+
mergeStates,
8
+
} from "../../../../hydration/index.ts";
1
9
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
3
-
import { FollowDocument } from "../../../../data-plane/db/models.ts";
4
-
import { ensureValidDid, isValidHandle } from "@atp/syntax";
5
-
import { RootFilterQuery } from "mongoose";
6
-
import { XRPCError } from "@atp/xrpc-server";
7
-
import { OutputSchema } from "../../../../lex/types/so/sprk/graph/getFollowers.ts";
10
+
import { QueryParams } from "../../../../lex/types/so/sprk/graph/getFollowers.ts";
8
11
import {
9
-
getProfileView,
10
-
getProfileViews,
11
-
} from "../../../../utils/profile-helper.ts";
12
+
createPipeline,
13
+
HydrationFnInput,
14
+
PresentationFnInput,
15
+
RulesFnInput,
16
+
SkeletonFnInput,
17
+
} from "../../../../pipeline.ts";
18
+
import { uriToDid as didFromUri } from "../../../../utils/uris.ts";
19
+
import { Views } from "../../../../views/index.ts";
20
+
import { clearlyBadCursor, resHeaders } from "../../../util.ts";
12
21
13
22
export default function (server: Server, ctx: AppContext) {
23
+
const getFollowers = createPipeline(
24
+
skeleton,
25
+
hydration,
26
+
noBlocks,
27
+
presentation,
28
+
);
14
29
server.so.sprk.graph.getFollowers({
15
-
auth: ctx.authVerifier.standardOptional,
30
+
auth: ctx.authVerifier.optionalStandardOrRole,
16
31
handler: async ({ params, auth }) => {
17
-
const { actor } = params;
18
-
const limit = params.limit;
19
-
const cursor = params.cursor;
20
-
const viewerDid = auth.credentials.type === "standard"
21
-
? auth.credentials.iss
22
-
: undefined;
32
+
const { viewer, includeTakedowns } = ctx.authVerifier.parseCreds(auth);
33
+
const hydrateCtx = ctx.hydrator.createContext({
34
+
viewer,
35
+
includeTakedowns,
36
+
});
37
+
38
+
const result = await getFollowers({ ...params, hydrateCtx }, ctx);
39
+
40
+
return {
41
+
encoding: "application/json",
42
+
body: result,
43
+
headers: resHeaders({}),
44
+
};
45
+
},
46
+
});
47
+
}
23
48
24
-
let actorDid;
49
+
const skeleton = async (input: SkeletonFnInput<Context, Params>) => {
50
+
const { params, ctx } = input;
51
+
const [subjectDid] = await ctx.hydrator.actor.getDidsDefined([params.actor]);
52
+
if (!subjectDid) {
53
+
throw new InvalidRequestError(`Actor not found: ${params.actor}`);
54
+
}
55
+
if (clearlyBadCursor(params.cursor)) {
56
+
return { subjectDid, followUris: [] };
57
+
}
58
+
const { followers, cursor } = await ctx.hydrator.graph.getActorFollowers({
59
+
did: subjectDid,
60
+
cursor: params.cursor,
61
+
limit: params.limit,
62
+
});
63
+
return {
64
+
subjectDid,
65
+
followUris: followers.map((f) => f.uri),
66
+
cursor: cursor || undefined,
67
+
};
68
+
};
25
69
26
-
if (isValidHandle(actor)) {
27
-
const actorDidDoc = await ctx.resolver.resolveHandleToDidDoc(actor);
28
-
actorDid = actorDidDoc.did;
29
-
} else {
30
-
try {
31
-
ensureValidDid(actor);
32
-
actorDid = actor;
33
-
} catch (error) {
34
-
ctx.logger.warn(
35
-
"Failed to ensure valid DID",
36
-
{ did: actor, error: (error as Error).message },
37
-
);
38
-
throw new XRPCError(400, "Invalid actor DID");
39
-
}
70
+
const hydration = async (
71
+
input: HydrationFnInput<Context, Params, SkeletonState>,
72
+
) => {
73
+
const { ctx, params, skeleton } = input;
74
+
const { followUris, subjectDid } = skeleton;
75
+
const followState = await ctx.hydrator.hydrateFollows(
76
+
followUris,
77
+
);
78
+
const dids = [subjectDid];
79
+
if (followState.follows) {
80
+
for (const [uri, follow] of followState.follows) {
81
+
if (follow) {
82
+
dids.push(didFromUri(uri));
40
83
}
84
+
}
85
+
}
86
+
const profileState = await ctx.hydrator.hydrateProfiles(
87
+
dids,
88
+
params.hydrateCtx,
89
+
);
90
+
return mergeStates(followState, profileState);
91
+
};
41
92
42
-
const query: RootFilterQuery<FollowDocument> = {
43
-
subject: actorDid,
44
-
};
93
+
const noBlocks = (input: RulesFnInput<Context, Params, SkeletonState>) => {
94
+
const { skeleton, params, hydration, ctx } = input;
95
+
const viewer = params.hydrateCtx.viewer;
96
+
skeleton.followUris = skeleton.followUris.filter((followUri) => {
97
+
const followerDid = didFromUri(followUri);
98
+
return (
99
+
!hydration.followBlocks?.get(followUri) &&
100
+
(!viewer || !ctx.views.viewerBlockExists(followerDid, hydration))
101
+
);
102
+
});
103
+
return skeleton;
104
+
};
45
105
46
-
if (cursor) {
47
-
query._id = { $gt: cursor };
48
-
}
106
+
const presentation = (
107
+
input: PresentationFnInput<Context, Params, SkeletonState>,
108
+
) => {
109
+
const { ctx, hydration, skeleton, params } = input;
110
+
const { subjectDid, followUris, cursor } = skeleton;
111
+
const isNoHosted = (did: string) => ctx.views.actorIsNoHosted(did, hydration);
49
112
50
-
// Get followers with pagination and subject profile concurrently
51
-
const [followers, subjectProfileView] = await Promise.all([
52
-
ctx.db.models.Follow.find(query)
53
-
.sort({ _id: 1 })
54
-
.limit(limit)
55
-
.lean(),
56
-
getProfileView(ctx, actorDid, viewerDid),
57
-
]);
113
+
const subject = ctx.views.profile(subjectDid, hydration);
114
+
if (
115
+
!subject ||
116
+
(!params.hydrateCtx.includeTakedowns && isNoHosted(subjectDid))
117
+
) {
118
+
throw new InvalidRequestError(`Actor not found: ${params.actor}`);
119
+
}
58
120
59
-
// Get next cursor
60
-
const nextCursor = followers.length === limit
61
-
? followers[followers.length - 1]._id.toString()
62
-
: undefined;
121
+
const followers = mapDefined(followUris, (followUri) => {
122
+
const followerDid = didFromUri(followUri);
123
+
if (!params.hydrateCtx.includeTakedowns && isNoHosted(followerDid)) {
124
+
return;
125
+
}
126
+
return ctx.views.profile(didFromUri(followUri), hydration);
127
+
});
63
128
64
-
// Extract follower DIDs and batch fetch profile views
65
-
const followerDids = followers.map((follow) => follow.authorDid);
66
-
const profileViews = await getProfileViews(ctx, followerDids, viewerDid);
129
+
return { followers, subject, cursor };
130
+
};
67
131
68
-
const res = {
69
-
encoding: "application/json",
70
-
body: {
71
-
subject: subjectProfileView,
72
-
followers: profileViews,
73
-
cursor: nextCursor,
74
-
} satisfies OutputSchema,
75
-
} as const;
132
+
type Context = {
133
+
hydrator: Hydrator;
134
+
views: Views;
135
+
};
76
136
77
-
return res;
78
-
},
79
-
});
80
-
}
137
+
type Params = QueryParams & {
138
+
hydrateCtx: HydrateCtx;
139
+
};
140
+
141
+
type SkeletonState = {
142
+
subjectDid: string;
143
+
followUris: string[];
144
+
cursor?: string;
145
+
};
+133
-68
api/so/sprk/graph/getFollows.ts
+133
-68
api/so/sprk/graph/getFollows.ts
···
1
+
import { mapDefined } from "@atp/common";
2
+
import { InvalidRequestError } from "@atp/xrpc-server";
3
+
import { AppContext } from "../../../../context.ts";
4
+
import {
5
+
HydrateCtx,
6
+
Hydrator,
7
+
mergeStates,
8
+
} from "../../../../hydration/index.ts";
1
9
import { Server } from "../../../../lex/index.ts";
2
-
import { FollowDocument } from "../../../../data-plane/db/models.ts";
3
-
import { AppContext } from "../../../../main.ts";
4
-
import { ensureValidDid, isValidHandle } from "@atp/syntax";
5
-
import { RootFilterQuery } from "mongoose";
6
-
import { XRPCError } from "@atp/xrpc-server";
7
-
import { OutputSchema } from "../../../../lex/types/so/sprk/graph/getFollows.ts";
10
+
import { QueryParams } from "../../../../lex/types/so/sprk/graph/getFollowers.ts";
8
11
import {
9
-
getProfileView,
10
-
getProfileViews,
11
-
} from "../../../../utils/profile-helper.ts";
12
+
createPipeline,
13
+
HydrationFnInput,
14
+
PresentationFnInput,
15
+
RulesFnInput,
16
+
SkeletonFnInput,
17
+
} from "../../../../pipeline.ts";
18
+
import { Views } from "../../../../views/index.ts";
19
+
import { clearlyBadCursor, resHeaders } from "../../../util.ts";
12
20
13
21
export default function (server: Server, ctx: AppContext) {
22
+
const getFollows = createPipeline(
23
+
skeleton,
24
+
hydration,
25
+
noBlocks,
26
+
presentation,
27
+
);
14
28
server.so.sprk.graph.getFollows({
15
-
auth: ctx.authVerifier.standardOptional,
29
+
auth: ctx.authVerifier.optionalStandardOrRole,
16
30
handler: async ({ params, auth }) => {
17
-
const { actor } = params;
18
-
const limit = params.limit;
19
-
const cursor = params.cursor;
20
-
const viewerDid = auth.credentials.type === "standard"
21
-
? auth.credentials.iss
22
-
: undefined;
31
+
const { viewer, includeTakedowns } = ctx.authVerifier.parseCreds(auth);
32
+
const hydrateCtx = ctx.hydrator.createContext({
33
+
viewer,
34
+
includeTakedowns,
35
+
});
36
+
37
+
// @TODO ensure canViewTakedowns gets threaded through and applied properly
38
+
const result = await getFollows({ ...params, hydrateCtx }, ctx);
39
+
40
+
return {
41
+
encoding: "application/json",
42
+
body: result,
43
+
headers: resHeaders({}),
44
+
};
45
+
},
46
+
});
47
+
}
23
48
24
-
let actorDid;
49
+
const skeleton = async (input: SkeletonFnInput<Context, Params>) => {
50
+
const { params, ctx } = input;
51
+
const [subjectDid] = await ctx.hydrator.actor.getDidsDefined([params.actor]);
52
+
if (!subjectDid) {
53
+
throw new InvalidRequestError(`Actor not found: ${params.actor}`);
54
+
}
55
+
if (clearlyBadCursor(params.cursor)) {
56
+
return { subjectDid, followUris: [] };
57
+
}
58
+
const { follows, cursor } = await ctx.hydrator.graph.getActorFollows({
59
+
did: subjectDid,
60
+
cursor: params.cursor,
61
+
limit: params.limit,
62
+
});
63
+
return {
64
+
subjectDid,
65
+
followUris: follows.map((f) => f.uri),
66
+
cursor: cursor || undefined,
67
+
};
68
+
};
25
69
26
-
if (isValidHandle(actor)) {
27
-
const actorDidDoc = await ctx.resolver.resolveHandleToDidDoc(actor);
28
-
actorDid = actorDidDoc.did;
29
-
} else {
30
-
try {
31
-
ensureValidDid(actor);
32
-
actorDid = actor;
33
-
} catch (error) {
34
-
ctx.logger.warn(
35
-
"Failed to ensure valid DID",
36
-
{ did: actor, error: (error as Error).message },
37
-
);
38
-
throw new XRPCError(400, "Invalid actor DID");
39
-
}
70
+
const hydration = async (
71
+
input: HydrationFnInput<Context, Params, SkeletonState>,
72
+
) => {
73
+
const { ctx, params, skeleton } = input;
74
+
const { followUris, subjectDid } = skeleton;
75
+
const followState = await ctx.hydrator.hydrateFollows(
76
+
followUris,
77
+
);
78
+
const dids = [subjectDid];
79
+
if (followState.follows) {
80
+
for (const follow of followState.follows.values()) {
81
+
if (follow) {
82
+
dids.push(follow.record.subject);
40
83
}
84
+
}
85
+
}
86
+
const profileState = await ctx.hydrator.hydrateProfiles(
87
+
dids,
88
+
params.hydrateCtx,
89
+
);
90
+
console.log(profileState);
91
+
return mergeStates(followState, profileState);
92
+
};
41
93
42
-
const query: RootFilterQuery<FollowDocument> = {
43
-
authorDid: actorDid,
44
-
};
94
+
const noBlocks = (input: RulesFnInput<Context, Params, SkeletonState>) => {
95
+
const { skeleton, params, hydration, ctx } = input;
96
+
const viewer = params.hydrateCtx.viewer;
97
+
skeleton.followUris = skeleton.followUris.filter((followUri) => {
98
+
const follow = hydration.follows?.get(followUri);
99
+
if (!follow) return false;
100
+
return (
101
+
!hydration.followBlocks?.get(followUri) &&
102
+
(!viewer ||
103
+
!ctx.views.viewerBlockExists(follow.record.subject, hydration))
104
+
);
105
+
});
106
+
return skeleton;
107
+
};
45
108
46
-
if (cursor) {
47
-
query._id = { $gt: cursor };
48
-
}
109
+
const presentation = (
110
+
input: PresentationFnInput<Context, Params, SkeletonState>,
111
+
) => {
112
+
const { ctx, hydration, skeleton, params } = input;
113
+
const { subjectDid, followUris, cursor } = skeleton;
114
+
const isNoHosted = (did: string) => ctx.views.actorIsNoHosted(did, hydration);
49
115
50
-
// Get follows with pagination and subject profile concurrently
51
-
const [follows, subjectProfileView] = await Promise.all([
52
-
ctx.db.models.Follow.find(query)
53
-
.sort({ _id: 1 })
54
-
.limit(limit)
55
-
.lean(),
56
-
getProfileView(ctx, actorDid, viewerDid),
57
-
]);
116
+
const subject = ctx.views.profile(subjectDid, hydration);
117
+
if (
118
+
!subject ||
119
+
(!params.hydrateCtx.includeTakedowns && isNoHosted(subjectDid))
120
+
) {
121
+
throw new InvalidRequestError(`Actor not found: ${params.actor}`);
122
+
}
58
123
59
-
// Get next cursor
60
-
const nextCursor = follows.length === limit
61
-
? follows[follows.length - 1]._id.toString()
62
-
: undefined;
124
+
const follows = mapDefined(followUris, (followUri) => {
125
+
const followDid = hydration.follows?.get(followUri)?.record.subject;
126
+
if (!followDid) return;
127
+
if (!params.hydrateCtx.includeTakedowns && isNoHosted(followDid)) {
128
+
return;
129
+
}
130
+
return ctx.views.profile(followDid, hydration);
131
+
});
63
132
64
-
// Extract follow subject DIDs and batch fetch profile views
65
-
const followSubjectDids = follows.map((follow) => follow.subject);
66
-
const profileViews = await getProfileViews(
67
-
ctx,
68
-
followSubjectDids,
69
-
viewerDid,
70
-
);
133
+
return { follows, subject, cursor };
134
+
};
71
135
72
-
const res = {
73
-
encoding: "application/json",
74
-
body: {
75
-
subject: subjectProfileView,
76
-
follows: profileViews,
77
-
cursor: nextCursor,
78
-
} satisfies OutputSchema,
79
-
} as const;
136
+
type Context = {
137
+
hydrator: Hydrator;
138
+
views: Views;
139
+
};
140
+
141
+
type Params = QueryParams & {
142
+
hydrateCtx: HydrateCtx;
143
+
};
80
144
81
-
return res;
82
-
},
83
-
});
84
-
}
145
+
type SkeletonState = {
146
+
subjectDid: string;
147
+
followUris: string[];
148
+
cursor?: string;
149
+
};
+2
-2
api/so/sprk/sound/getActorAudios.ts
+2
-2
api/so/sprk/sound/getActorAudios.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { transformAudiosToAudioViews } from "../../../../utils/audio-transformer.ts";
4
4
import { decodeBase64, encodeBase64 } from "@std/encoding";
5
5
···
37
37
let actorDid = actor;
38
38
if (!actor.startsWith("did:")) {
39
39
try {
40
-
const didDoc = await ctx.resolver.resolveHandleToDidDoc(actor);
40
+
const didDoc = await ctx.idResolver.did.resolveAtprotoData(actor);
41
41
actorDid = didDoc.did;
42
42
} catch (err) {
43
43
console.error("Failed to resolve handle:", err);
+1
-1
api/so/sprk/sound/getAudioPosts.ts
+1
-1
api/so/sprk/sound/getAudioPosts.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { transformPostsToPostViews } from "../../../../utils/post-transformer.ts";
4
4
import { decodeBase64, encodeBase64 } from "@std/encoding";
5
5
import { transformAudioToAudioView } from "../../../../utils/audio-transformer.ts";
+1
-1
api/so/sprk/sound/getAudios.ts
+1
-1
api/so/sprk/sound/getAudios.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { transformAudiosToAudioViews } from "../../../../utils/audio-transformer.ts";
4
4
import { AudioDocument } from "../../../../data-plane/db/models.ts";
5
5
+1
-1
api/so/sprk/sound/getTrendingAudios.ts
+1
-1
api/so/sprk/sound/getTrendingAudios.ts
···
1
1
import { Server } from "../../../../lex/index.ts";
2
-
import { AppContext } from "../../../../main.ts";
2
+
import { AppContext } from "../../../../context.ts";
3
3
import { transformAudiosToAudioViews } from "../../../../utils/audio-transformer.ts";
4
4
import { AudioDocument } from "../../../../data-plane/db/models.ts";
5
5
+31
-35
api/well-known.ts
+31
-35
api/well-known.ts
···
1
1
import { Hono } from "hono";
2
-
import { env } from "../utils/env.ts";
3
2
import { formatMultikey, Secp256k1Keypair } from "@atp/crypto";
3
+
import { AppEnv } from "../context.ts";
4
4
5
-
const wellKnownRouter = () => {
6
-
const router = new Hono();
5
+
const app = new Hono<AppEnv>();
7
6
8
-
router.get("/.well-known/did.json", async (c) => {
9
-
const domain = env.PUBLIC_URL?.split("://")[1] || "localhost";
10
-
const keypair = await Secp256k1Keypair.import(
11
-
Deno.env.get("APPVIEW_K256_PRIVATE_KEY_HEX") || "",
12
-
);
13
-
const multikey = formatMultikey(keypair.jwtAlg, keypair.publicKeyBytes());
7
+
app.get("/did.json", (c) => {
8
+
const did = c.env.cfg.serverDid;
9
+
const keypair = Secp256k1Keypair.import(
10
+
c.env.cfg.privateKey,
11
+
);
12
+
const multikey = formatMultikey(keypair.jwtAlg, keypair.publicKeyBytes());
14
13
15
-
return c.json({
16
-
"@context": [
17
-
"https://www.w3.org/ns/did/v1",
18
-
"https://w3id.org/security/multikey/v1",
19
-
],
20
-
id: `did:web:${domain}`,
21
-
verificationMethod: [
22
-
{
23
-
id: `did:web:${domain}#atproto`,
24
-
type: "Multikey",
25
-
controller: `did:web:${domain}`,
26
-
publicKeyMultibase: multikey,
27
-
},
28
-
],
29
-
service: [
30
-
{
31
-
id: "#sprk_appview",
32
-
type: "SprkAppView",
33
-
serviceEndpoint: `https://${domain}`,
34
-
},
35
-
],
36
-
});
14
+
return c.json({
15
+
"@context": [
16
+
"https://www.w3.org/ns/did/v1",
17
+
"https://w3id.org/security/multikey/v1",
18
+
],
19
+
id: did,
20
+
verificationMethod: [
21
+
{
22
+
id: `${did}#atproto`,
23
+
type: "Multikey",
24
+
controller: did,
25
+
publicKeyMultibase: multikey,
26
+
},
27
+
],
28
+
service: [
29
+
{
30
+
id: "#sprk_appview",
31
+
type: "SprkAppView",
32
+
serviceEndpoint: c.env.cfg.publicUrl,
33
+
},
34
+
],
37
35
});
38
-
39
-
return router;
40
-
};
36
+
});
41
37
42
-
export default wellKnownRouter;
38
+
export default app;
+8
-6
auth-verifier.ts
+8
-6
auth-verifier.ts
···
505
505
auth: StandardOutput | RoleOutput | NullOutput | ModServiceOutput,
506
506
) {
507
507
const creds = auth.credentials;
508
-
const isAdmin = creds.type === "role" && creds.admin;
509
-
const isModService =
510
-
(creds.type === "standard" || creds.type === "mod_service") &&
511
-
creds.iss && this.isModService(creds.iss);
512
-
const includeTakedownsAnd3pBlocks = Boolean(isAdmin || isModService);
508
+
const includeTakedownsAnd3pBlocks =
509
+
(creds.type === "role" && creds.admin) ||
510
+
creds.type === "mod_service" ||
511
+
(creds.type === "standard" &&
512
+
this.isModService(creds.iss));
513
+
const canPerformTakedown = (creds.type === "role" && creds.admin) ||
514
+
creds.type === "mod_service";
513
515
return {
514
516
viewer: creds.type === "standard" ? creds.iss : null,
515
517
includeTakedowns: includeTakedownsAnd3pBlocks,
516
518
include3pBlocks: includeTakedownsAnd3pBlocks,
517
-
canPerformTakedown: includeTakedownsAnd3pBlocks,
519
+
canPerformTakedown,
518
520
};
519
521
}
520
522
}
+52
-28
compose.dev.yaml
+52
-28
compose.dev.yaml
···
9
9
- "27017:27017"
10
10
volumes:
11
11
- ./devdb:/data/db
12
-
# Generate with `openssl rand -base64 756 > mongo-keyfile && chmod 400 mongo-keyfile && sudo chown 999:999 mongo-keyfile`
13
-
- ./mongo-keyfile:/etc/mongo-keyfile:ro
14
-
command: ["--replSet", "rs0", "--keyFile", "/etc/mongo-keyfile", "--auth"]
15
12
healthcheck:
16
13
test: ["CMD", "mongosh", "--eval", '''db.adminCommand("ping")''']
17
14
interval: 10s
···
19
16
retries: 3
20
17
restart: unless-stopped
21
18
22
-
mongo-init-replica:
23
-
image: mongo:8
19
+
app:
20
+
build:
21
+
context: .
22
+
dockerfile: Dockerfile
23
+
command: ["deno", "run", "-A", "--watch", "main.ts"]
24
+
environment:
25
+
NODE_ENV: development
26
+
HOST: 0.0.0.0
27
+
SPRK_PORT: 3000
28
+
SPRK_DB_URI: mongodb://mongo:mongo@db:27017
29
+
SPRK_DB_NAME: dev
30
+
SPRK_ADMIN_PASSWORDS: "12345"
31
+
env_file:
32
+
- .env
33
+
ports:
34
+
- "4000:3000"
24
35
depends_on:
25
36
db:
26
37
condition: service_healthy
27
-
entrypoint: >
28
-
bash -c "
29
-
echo 'initiating replica set...';
30
-
mongosh --host db:27017 -u mongo -p mongo --eval '
31
-
rs.initiate({
32
-
_id: \"rs0\",
33
-
members: [{ _id: 0, host: \"db:27017\" }]
34
-
})
35
-
';
36
-
echo 'replica set ready';
37
-
"
38
-
restart: "no"
38
+
develop:
39
+
watch:
40
+
- path: ./api
41
+
action: sync
42
+
target: /app/api
43
+
- path: ./lexicon
44
+
action: sync
45
+
target: /app/lexicon
46
+
- path: ./services
47
+
action: sync
48
+
target: /app/services
49
+
- path: ./utils
50
+
action: sync
51
+
target: /app/utils
52
+
- path: ./data-plane
53
+
action: sync
54
+
target: /app/data-plane
55
+
- path: ./main.ts
56
+
action: sync
57
+
target: /app/main.ts
58
+
- path: ./views
59
+
action: sync
60
+
target: /app/views
61
+
- path: ./data-plane
62
+
action: sync
63
+
target: /app/data-plane
64
+
- path: ./hydration
65
+
action: sync
66
+
target: /app/hydration
67
+
restart: unless-stopped
39
68
40
-
app:
69
+
ingester:
41
70
build:
42
71
context: .
43
-
dockerfile: Dockerfile.dev
72
+
dockerfile: Dockerfile
73
+
command: ["deno", "run", "-A", "--watch", "ingest.ts"]
44
74
environment:
45
75
NODE_ENV: development
46
76
HOST: 0.0.0.0
47
-
PORT: 3000
48
-
DB_HOST: db
49
-
DB_PORT: 27017
50
-
DB_USER: mongo
51
-
DB_PASSWORD: mongo
52
-
DB_NAME: dev
53
-
ADMIN_PASSWORD: "00000000000000000000000000000000"
77
+
SPRK_DB_URI: mongodb://mongo:mongo@db:27017
78
+
SPRK_DB_NAME: dev
79
+
ADMIN_PASSWORD: "12345"
54
80
env_file:
55
81
- .env
56
-
ports:
57
-
- "4000:3000"
58
82
depends_on:
59
83
db:
60
84
condition: service_healthy
+168
config.ts
+168
config.ts
···
1
+
import * as dotenv from "dotenv";
2
+
import { envInt, envList, envStr } from "@atp/common";
3
+
4
+
dotenv.config({ quiet: true });
5
+
6
+
export interface ServerConfigValues {
7
+
version?: string;
8
+
debugMode?: boolean;
9
+
port?: number;
10
+
publicUrl?: string;
11
+
serverDid: string;
12
+
privateKey: string;
13
+
alternateAudienceDids: string[];
14
+
modServiceDid: string;
15
+
adminPasswords: string[];
16
+
indexedAtEpoch?: Date;
17
+
18
+
bigThreadUris: Set<string>;
19
+
bigThreadDepth?: number;
20
+
maxThreadDepth?: number;
21
+
maxThreadParents: number;
22
+
23
+
videoCdn?: string;
24
+
hlsCdn?: string;
25
+
mediaCdn?: string;
26
+
thumbCdn?: string;
27
+
28
+
dbUri?: string;
29
+
dbName?: string;
30
+
relayUrl?: string;
31
+
plcUrl?: string;
32
+
}
33
+
34
+
export class ServerConfig {
35
+
constructor(private cfg: ServerConfigValues) {}
36
+
37
+
static readEnv() {
38
+
const version = envStr("SPRK_VERSION");
39
+
const debugMode = Deno.env.get("NODE_ENV") === "development" ||
40
+
Deno.env.get("NODE_ENV") === "test";
41
+
const port = envInt("SPRK_PORT") ?? 3000;
42
+
const publicUrl = envStr("SPRK_PUBLIC_URL") ?? undefined;
43
+
const serverDid = envStr("SPRK_SERVER_DID") ?? "did:example:test";
44
+
const privateKey = envStr("SPRK_PRIVATE_KEY") ?? "";
45
+
const alternateAudienceDids = envList("SPRK_ALTERNATE_AUDIENCE_DIDS") ?? [];
46
+
const modServiceDid = envStr("SPRK_MOD_SERVICE_DID") ?? "did:web:localhost";
47
+
const adminPasswords = envList("SPRK_ADMIN_PASSWORDS") ?? ["admin-token"];
48
+
49
+
const indexedAtEpochEnv = Deno.env.get("SPRK_INDEXED_AT_EPOCH");
50
+
const indexedAtEpoch = indexedAtEpochEnv !== undefined
51
+
? new Date(indexedAtEpochEnv)
52
+
: undefined;
53
+
54
+
const bigThreadUris = new Set(envList("SPRK_BIG_THREAD_URIS"));
55
+
const bigThreadDepth = envInt("SPRK_BIG_THREAD_DEPTH") ?? 10;
56
+
const maxThreadDepth = envInt("SPRK_MAX_THREAD_DEPTH") ?? 10;
57
+
const maxThreadParents = envInt("SPRK_MAX_THREAD_PARENTS") ?? 10;
58
+
59
+
const videoCdn = envStr("SPRK_VIDEO_CDN") ?? "https://video.sprk.so";
60
+
const hlsCdn = envStr("SPRK_HLS_CDN") ?? "https://hls.sprk.so";
61
+
const mediaCdn = envStr("SPRK_MEDIA_CDN") ?? "https://media.sprk.so";
62
+
const thumbCdn = envStr("SPRK_THUMB_CDN") ?? "https://thumb.sprk.so";
63
+
64
+
const dbUri = envStr("SPRK_DB_URI") ??
65
+
"mongodb://mongo:mongo@localhost:27017/dev";
66
+
const dbName = envStr("SPRK_DB_NAME") ?? "dev";
67
+
const relayUrl = envStr("SPRK_RELAY") ??
68
+
"wss://relay1.us-east.bsky.network";
69
+
const plcUrl = envStr("SPRK_PLC") ?? "https://plc.directory";
70
+
71
+
return new ServerConfig({
72
+
version,
73
+
debugMode,
74
+
port,
75
+
publicUrl,
76
+
serverDid,
77
+
privateKey,
78
+
alternateAudienceDids,
79
+
modServiceDid,
80
+
adminPasswords,
81
+
indexedAtEpoch,
82
+
bigThreadUris,
83
+
bigThreadDepth,
84
+
maxThreadDepth,
85
+
maxThreadParents,
86
+
videoCdn,
87
+
hlsCdn,
88
+
mediaCdn,
89
+
thumbCdn,
90
+
dbUri,
91
+
dbName,
92
+
relayUrl,
93
+
plcUrl,
94
+
});
95
+
}
96
+
97
+
get version() {
98
+
return this.cfg.version;
99
+
}
100
+
get debugMode() {
101
+
return this.cfg.debugMode;
102
+
}
103
+
get port() {
104
+
return this.cfg.port;
105
+
}
106
+
get publicUrl() {
107
+
return this.cfg.publicUrl;
108
+
}
109
+
get serverDid() {
110
+
return this.cfg.serverDid;
111
+
}
112
+
get privateKey() {
113
+
return this.cfg.privateKey;
114
+
}
115
+
get alternateAudienceDids() {
116
+
return this.cfg.alternateAudienceDids;
117
+
}
118
+
get modServiceDid() {
119
+
return this.cfg.modServiceDid;
120
+
}
121
+
get adminPasswords() {
122
+
return this.cfg.adminPasswords;
123
+
}
124
+
get indexedAtEpoch() {
125
+
return this.cfg.indexedAtEpoch;
126
+
}
127
+
128
+
// Threads
129
+
get bigThreadDepth() {
130
+
return this.cfg.bigThreadDepth;
131
+
}
132
+
get bigThreadUris() {
133
+
return this.cfg.bigThreadUris;
134
+
}
135
+
get maxThreadParents() {
136
+
return this.cfg.maxThreadParents;
137
+
}
138
+
get maxThreadDepth() {
139
+
return this.cfg.maxThreadDepth;
140
+
}
141
+
142
+
// CDNs
143
+
get videoCdn() {
144
+
return this.cfg.videoCdn;
145
+
}
146
+
get hlsCdn() {
147
+
return this.cfg.hlsCdn;
148
+
}
149
+
get mediaCdn() {
150
+
return this.cfg.mediaCdn;
151
+
}
152
+
get thumbCdn() {
153
+
return this.cfg.thumbCdn;
154
+
}
155
+
156
+
get dbUri() {
157
+
return this.cfg.dbUri;
158
+
}
159
+
get dbName() {
160
+
return this.cfg.dbName;
161
+
}
162
+
get relayUrl() {
163
+
return this.cfg.relayUrl;
164
+
}
165
+
get plcUrl() {
166
+
return this.cfg.plcUrl;
167
+
}
168
+
}
+27
context.ts
+27
context.ts
···
1
+
import { Logger } from "@logtape/logtape";
2
+
import { Database } from "./data-plane/db/index.ts";
3
+
import { DataPlane } from "./data-plane/index.ts";
4
+
import { Hydrator } from "./hydration/index.ts";
5
+
import { Views } from "./views/index.ts";
6
+
import { IdResolver } from "@atp/identity";
7
+
import { AuthVerifier } from "./auth-verifier.ts";
8
+
import { ServerConfig } from "./config.ts";
9
+
10
+
export type AppContext = {
11
+
db: Database;
12
+
dataplane: DataPlane;
13
+
hydrator: Hydrator;
14
+
views: Views;
15
+
logger: Logger;
16
+
idResolver: IdResolver;
17
+
authVerifier: AuthVerifier;
18
+
cfg: ServerConfig;
19
+
};
20
+
21
+
export type AppEnv = {
22
+
Bindings: AppContext;
23
+
Variables: {
24
+
did: string;
25
+
isAdmin: boolean;
26
+
};
27
+
};
+1
-2
data-plane/background.ts
+1
-2
data-plane/background.ts
···
1
1
import PQueue from "p-queue";
2
2
import { Database } from "./db/index.ts";
3
3
import { Logger } from "@logtape/logtape";
4
-
import { env } from "../utils/env.ts";
5
4
6
5
// A simple queue for in-process, out-of-band/backgrounded work
7
6
8
7
export class BackgroundQueue {
9
-
queue = new PQueue({ concurrency: env.BACKGROUND_CONCURRENCY });
8
+
queue = new PQueue();
10
9
destroyed = false;
11
10
private processAllInterval: number | null = null;
12
11
private isProcessingAll = false;
+9
-33
data-plane/db/index.ts
+9
-33
data-plane/db/index.ts
···
1
1
import mongoose, { Connection } from "mongoose";
2
2
import { IdResolver, MemoryCache } from "@atp/identity";
3
-
import { env } from "../../utils/env.ts";
4
3
import * as models from "./models.ts";
5
4
import { getResultFromDoc } from "../util.ts";
6
5
import { getLogger } from "@logtape/logtape";
6
+
import { ServerConfig } from "../../config.ts";
7
7
8
8
const HOUR = 60 * 60 * 1000;
9
9
const DAY = HOUR * 24;
···
14
14
public logger = getLogger(["appview", "database"]);
15
15
public idResolver: IdResolver;
16
16
17
-
constructor() {
17
+
constructor(private cfg: ServerConfig) {
18
18
this.idResolver = new IdResolver({
19
19
didCache: new MemoryCache(HOUR, DAY),
20
20
});
21
21
}
22
22
23
23
connect() {
24
-
const { DB_URI, DB_USER, DB_PASSWORD, DB_HOST, DB_PORT, DB_NAME } = env;
25
-
26
-
const uri = DB_URI ||
27
-
`mongodb://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/?appName=appview`;
28
-
29
-
this.logger.info(
30
-
DB_URI
31
-
? `Connecting to MongoDB using provided URI`
32
-
: `Connecting to MongoDB at ${DB_HOST}:${DB_PORT}/?appName=appview`,
33
-
);
24
+
const uri = this.cfg.dbUri;
25
+
if (!uri) {
26
+
throw new Error("No database URI provided");
27
+
}
28
+
this.logger.info(`Connecting to ${uri}`);
34
29
35
30
try {
36
31
this.connection = mongoose.createConnection(uri, {
37
32
autoIndex: true,
38
33
autoCreate: true,
39
-
dbName: DB_NAME,
40
-
maxPoolSize: env.MONGO_MAX_POOL_SIZE,
41
34
});
42
35
43
36
// Attach basic listeners for visibility
···
139
132
),
140
133
};
141
134
142
-
this.logger.info("Connected to MongoDB");
135
+
this.logger.info("Started connection to MongoDB");
143
136
} catch (error) {
144
-
this.logger.error("Failed to connect to MongoDB", { error });
137
+
this.logger.error("Failed to start connection to MongoDB", { error });
145
138
throw error;
146
139
}
147
-
}
148
-
149
-
isConnected(): boolean {
150
-
return !!this.connection && this.connection.readyState === 1; // 1 = connected
151
-
}
152
-
153
-
async waitForConnection(timeoutMs = 30000, pollMs = 250): Promise<boolean> {
154
-
if (this.isConnected()) return true;
155
-
const start = Date.now();
156
-
return await new Promise((resolve) => {
157
-
const check = () => {
158
-
if (this.isConnected()) return resolve(true);
159
-
if (Date.now() - start >= timeoutMs) return resolve(false);
160
-
setTimeout(check, pollMs);
161
-
};
162
-
check();
163
-
});
164
140
}
165
141
166
142
async disconnect(): Promise<void> {
+2
-5
data-plane/routes/follows.ts
+2
-5
data-plane/routes/follows.ts
···
40
40
41
41
async getFollowers(actorDid: string, limit = 50, cursor?: string) {
42
42
// Build query for followers (people who follow this actor)
43
-
const followersQuery = this.db.models.Follow.find({ subject: actorDid })
44
-
.populate("actor", "did handle indexedAt takedownRef upstreamStatus");
43
+
const followersQuery = this.db.models.Follow.find({ subject: actorDid });
45
44
46
45
// Apply pagination using TimeCidKeyset
47
46
const paginatedQuery = this.timeCidKeyset.paginate(followersQuery, {
48
47
limit,
49
48
cursor,
50
-
direction: "desc",
51
49
});
52
50
53
51
const followers = await paginatedQuery.exec();
···
74
72
75
73
async getFollows(actorDid: string, limit = 50, cursor?: string) {
76
74
// Build query for follows (people this actor follows)
77
-
const followsQuery = this.db.models.Follow.find({ authorDid: actorDid })
78
-
.populate("actor", "did handle indexedAt takedownRef upstreamStatus");
75
+
const followsQuery = this.db.models.Follow.find({ authorDid: actorDid });
79
76
80
77
// Apply pagination using TimeCidKeyset
81
78
const paginatedQuery = this.timeCidKeyset.paginate(followsQuery, {
+9
-24
data-plane/subscription.ts
+9
-24
data-plane/subscription.ts
···
5
5
import { Database } from "./db/index.ts";
6
6
import { IndexingService } from "./indexing/index.ts";
7
7
import { getLogger, Logger } from "@logtape/logtape";
8
-
import { env } from "../utils/env.ts";
8
+
import { ServerConfig } from "../config.ts";
9
9
10
10
export class RepoSubscription {
11
11
firehose: Firehose;
···
17
17
18
18
constructor(
19
19
public opts: {
20
-
service: string;
21
20
db: Database;
22
21
idResolver: IdResolver;
23
22
startCursor?: number;
23
+
cfg: ServerConfig;
24
24
},
25
25
) {
26
-
const { service, db, idResolver, startCursor } = opts;
26
+
const { db, idResolver, startCursor, cfg } = opts;
27
27
this.logger = getLogger(["appview", "subscription"]);
28
28
this.background = new BackgroundQueue(db, this.logger);
29
29
this.indexingSvc = new IndexingService(
···
35
35
36
36
const { runner, firehose } = createFirehose({
37
37
idResolver,
38
-
service,
38
+
service: cfg.relayUrl,
39
39
indexingSvc: this.indexingSvc,
40
40
logger: this.logger,
41
41
db,
···
45
45
this.firehose = firehose;
46
46
}
47
47
48
-
async start() {
49
-
const connected = await this.indexingSvc.db.waitForConnection(30000);
50
-
if (!connected) {
51
-
throw new Error(
52
-
"Failed to connect to database during subscription start",
53
-
);
54
-
}
48
+
start() {
55
49
this.logger.info("Starting firehose subscription");
56
50
this.firehoseRunning = true;
57
51
this.firehose.start();
···
59
53
60
54
async restart() {
61
55
await this.destroy();
62
-
const connected = await this.indexingSvc.db.waitForConnection(30000);
63
-
if (!connected) {
64
-
throw new Error(
65
-
"Failed to connect to database during subscription restart",
66
-
);
67
-
}
68
56
69
57
// Read fresh cursor from database
70
58
const savedCursor = await this.opts.db.getCursorState();
···
72
60
73
61
const { runner, firehose } = createFirehose({
74
62
idResolver: this.opts.idResolver,
75
-
service: this.opts.service,
63
+
service: this.opts.cfg.relayUrl,
76
64
indexingSvc: this.indexingSvc,
77
65
logger: this.logger,
78
66
db: this.opts.db,
···
80
68
});
81
69
this.runner = runner;
82
70
this.firehose = firehose;
83
-
await this.start();
71
+
this.start();
84
72
}
85
73
86
74
async processAll() {
···
95
83
this.firehoseRunning = false;
96
84
}
97
85
this.logger.info("Processing remaining runner tasks...");
98
-
if (env.NODE_ENV === "development") {
86
+
if (this.opts.cfg.debugMode) {
99
87
const timeoutMs = 10000;
100
88
// Runner destroy with timeout and proper timer cleanup
101
89
let destroyTimeoutId: number | undefined;
···
150
138
151
139
function createFirehose(opts: {
152
140
idResolver: IdResolver;
153
-
service: string;
141
+
service?: string;
154
142
indexingSvc: IndexingService;
155
143
logger: Logger;
156
144
db: Database;
···
158
146
}): { firehose: Firehose; runner: MemoryRunner } {
159
147
const { idResolver, service, indexingSvc, logger, db, startCursor } = opts;
160
148
161
-
logger.info("Creating firehose subscription", { service, startCursor });
162
-
163
149
const runner = new MemoryRunner({
164
150
startCursor,
165
-
concurrency: env.RUNNER_CONCURRENCY,
166
151
setCursorInterval: 30000, // Save cursor every 30 seconds
167
152
setCursor: async (cursor: number) => {
168
153
await db.saveCursorState(cursor);
+6
-1
deno.json
+6
-1
deno.json
···
1
1
{
2
2
"tasks": {
3
-
"dev": "deno run -A --watch main.ts",
3
+
"dev:db": "mongod --dbpath ./devdb",
4
+
"dev:api": "deno run -A --watch main.ts",
5
+
"dev:ingest": "deno run -A --watch ingest.ts",
6
+
"dev": {
7
+
"dependencies": ["dev:db", "dev:api", "dev:ingest"]
8
+
},
4
9
"codegen": "deno run -A jsr:@atp/lex-cli@^0.1.0-alpha.1 gen-server -o ./lex -i ./lexicons",
5
10
"start": "deno run -A --env-file main.ts",
6
11
"docker-dev": "docker compose -f compose.dev.yaml up --build --watch"
+29
ingest.ts
+29
ingest.ts
···
1
+
import { RepoSubscription } from "./data-plane/subscription.ts";
2
+
import { IdResolver } from "@atp/identity";
3
+
import { ServerConfig } from "./config.ts";
4
+
import { Database } from "./data-plane/db/index.ts";
5
+
import { getLogger } from "@logtape/logtape";
6
+
import { configureLogger } from "./utils/logger.ts";
7
+
8
+
await configureLogger();
9
+
10
+
const logger = getLogger(["ingester"]);
11
+
const cfg = ServerConfig.readEnv();
12
+
13
+
const idResolver = new IdResolver({ plcUrl: cfg.plcUrl });
14
+
const db = new Database(cfg);
15
+
db.connect();
16
+
17
+
const savedCursor = cfg.debugMode ? null : await db.getCursorState();
18
+
const startCursor = savedCursor !== null ? savedCursor : undefined;
19
+
20
+
const sub = new RepoSubscription({
21
+
cfg,
22
+
db,
23
+
idResolver,
24
+
startCursor,
25
+
});
26
+
await sub.indexingSvc.indexRepo("did:plc:6hbqm2oftpotwuw7gvvrui3i");
27
+
28
+
sub.start();
29
+
logger.info("Subscription started");
+5
-1
lexicons/com/atproto/repo/getRecord.json
+5
-1
lexicons/com/atproto/repo/getRecord.json
···
19
19
"format": "nsid",
20
20
"description": "The NSID of the record collection."
21
21
},
22
-
"rkey": { "type": "string", "description": "The Record Key." },
22
+
"rkey": {
23
+
"type": "string",
24
+
"description": "The Record Key.",
25
+
"format": "record-key"
26
+
},
23
27
"cid": {
24
28
"type": "string",
25
29
"format": "cid",
+36
-223
main.ts
+36
-223
main.ts
···
1
1
import { Hono } from "hono";
2
2
import { cors } from "hono/cors";
3
-
import { HTTPException } from "hono/http-exception";
4
3
import { logger } from "hono/logger";
5
4
import { Database } from "./data-plane/db/index.ts";
6
-
import { env } from "./utils/env.ts";
7
5
import { createAuthVerifier } from "./auth-verifier.ts";
8
6
import API from "./api/index.ts";
9
7
import { createServer } from "./lex/index.ts";
10
-
import {
11
-
createBidirectionalResolver,
12
-
createIdResolver,
13
-
} from "./utils/id-resolver.ts";
14
-
import { takedownFilterMiddleware } from "./services/takedown-filter.ts";
15
-
import wellKnownRouter from "./api/well-known.ts";
16
-
import { TakedownService } from "./services/takedown.ts";
17
-
import { BidirectionalResolver } from "./utils/id-resolver.ts";
18
-
import { DidResolver } from "@atp/identity";
19
-
import { AuthVerifier } from "./auth-verifier.ts";
20
-
import { AuthRequiredError } from "@atp/xrpc-server";
21
-
import { RepoSubscription } from "./data-plane/subscription.ts";
8
+
import wellKnown from "./api/well-known.ts";
9
+
import health from "./api/health.ts";
10
+
import { IdResolver } from "@atp/identity";
22
11
import { DataPlane } from "./data-plane/index.ts";
23
-
import { configure, getConsoleSink, getLogger, Logger } from "@logtape/logtape";
24
-
import { getPrettyFormatter } from "@logtape/pretty";
12
+
import { getLogger } from "@logtape/logtape";
13
+
import { configureLogger } from "./utils/logger.ts";
25
14
import { Hydrator } from "./hydration/index.ts";
26
15
import { Views } from "./views/index.ts";
16
+
import { AppContext, AppEnv } from "./context.ts";
17
+
import { ServerConfig } from "./config.ts";
27
18
28
-
await configure({
29
-
sinks: {
30
-
console: getConsoleSink({
31
-
formatter: getPrettyFormatter({
32
-
properties: true,
33
-
categoryStyle: "underline",
34
-
messageColor: "rgb(255, 255, 255)",
35
-
categoryColor: "rgb(255, 255, 255)",
36
-
messageStyle: "reset",
37
-
}),
38
-
}),
39
-
},
40
-
loggers: [
41
-
{ category: "appview", lowestLevel: "info", sinks: ["console"] },
42
-
{ category: ["logtape", "meta"], lowestLevel: "error", sinks: ["console"] },
43
-
],
44
-
});
45
-
46
-
export type AppContext = {
47
-
db: Database;
48
-
dataplane: DataPlane;
49
-
hydrator: Hydrator;
50
-
views: Views;
51
-
logger: Logger;
52
-
resolver: BidirectionalResolver;
53
-
serviceDid: string;
54
-
didResolver: DidResolver;
55
-
takedownService: TakedownService;
56
-
authVerifier: AuthVerifier;
57
-
sub: RepoSubscription;
58
-
};
59
-
60
-
export type AppEnv = {
61
-
Bindings: AppContext;
62
-
Variables: {
63
-
did: string;
64
-
isAdmin: boolean;
65
-
};
66
-
};
19
+
await configureLogger();
67
20
68
21
// Create app without starting services
69
22
export function createApp(ctx: AppContext): Hono<AppEnv> {
70
23
const app = new Hono<AppEnv>();
71
24
72
-
app.use("*", async (c, next) => {
73
-
await next();
74
-
if (c.res.status === 500) {
75
-
ctx.logger.error(c.error?.message!);
76
-
console.log(c.error);
77
-
}
78
-
});
79
-
80
25
app.use("*", cors());
81
26
app.use("*", logger());
82
27
app.use("*", async (c, next) => {
83
-
// Initialize c.env if it doesn't exist (for testing compatibility)
84
-
if (!c.env) {
85
-
c.env = {} as AppContext;
86
-
}
87
-
c.env.serviceDid = ctx.serviceDid;
88
-
c.env.didResolver = ctx.didResolver;
89
-
c.env.takedownService = ctx.takedownService;
90
-
c.env.authVerifier = ctx.authVerifier;
91
-
c.env.sub = ctx.sub;
28
+
c.env = ctx;
92
29
await next();
93
30
});
94
-
app.use("*", takedownFilterMiddleware);
95
31
96
32
// Lexicon/XRPC server and routers
97
33
const lexServer = createServer();
98
34
API(lexServer, ctx);
99
-
100
-
app.route("/", wellKnownRouter());
101
35
app.route("/", lexServer.xrpc.app);
102
36
103
-
// Root route
104
-
app.get("/", (c) => {
105
-
return c.text(
106
-
"✧･ﾟ: ✧･ﾟ:. ݁₊ ⊹ . ݁˖ . ݁ SPARK API . ݁₊ ⊹ . ݁˖ . ݁ :･ﾟ✧:･ﾟ✧",
107
-
);
108
-
});
109
-
110
-
// Health endpoint
111
-
app.get("/xrpc/_health", (c) => {
112
-
const version = Deno.env.get("COMMIT_SHA") ?? "unknown";
113
-
return c.json({ version });
114
-
});
115
-
116
-
// Error handling
117
-
app.onError((err, c) => {
118
-
if (err instanceof HTTPException) return err.getResponse();
119
-
120
-
// Handle AuthRequiredError from XRPC server
121
-
if (
122
-
err instanceof AuthRequiredError ||
123
-
err.constructor?.name === "AuthRequiredError"
124
-
) {
125
-
const authErr = err as AuthRequiredError;
126
-
return c.json({
127
-
error: authErr.message || "Authentication Required",
128
-
message: authErr.message || "Invalid or missing credentials",
129
-
}, 401);
130
-
}
131
-
132
-
ctx.logger.error({ err });
133
-
return c.json({
134
-
error: "Internal Server Error",
135
-
message: "An unexpected error occurred",
136
-
}, 500);
137
-
});
138
-
37
+
app.route("/.well-known", wellKnown);
38
+
app.route("/", health);
139
39
return app;
140
40
}
141
41
142
42
// Setup function to create context and app
143
-
export async function setupApp(): Promise<
144
-
{ app: Hono<AppEnv>; ctx: AppContext }
145
-
> {
43
+
export function setupApp(): { app: Hono<AppEnv>; ctx: AppContext } {
146
44
// Setup logger and database
147
45
const appLogger = getLogger(["appview"]);
148
-
const db = new Database();
46
+
const cfg = ServerConfig.readEnv();
47
+
const db = new Database(cfg);
149
48
db.connect();
150
49
151
-
// Wait for database connection
152
-
const connected = await db.waitForConnection(30000);
153
-
if (!connected) {
154
-
throw new Error("Failed to connect to database during startup");
155
-
}
156
-
157
-
// Read cursor from database (skip in dev environment)
158
-
const savedCursor = env.NODE_ENV === "development"
159
-
? null
160
-
: await db.getCursorState();
161
-
const startCursor = savedCursor !== null ? savedCursor : undefined;
162
-
163
-
appLogger.info("Database cursor loaded", {
164
-
cursor: startCursor,
165
-
isDev: env.NODE_ENV === "development",
166
-
skippedSavedCursor: env.NODE_ENV === "development",
167
-
});
168
-
169
50
// DID and resolver setup
170
-
const baseIdResolver = createIdResolver();
171
-
const resolver = createBidirectionalResolver(baseIdResolver);
172
-
const serviceDid = env.SERVICE_DID;
51
+
const idResolver = new IdResolver({ plcUrl: cfg.plcUrl });
173
52
174
-
const dataplane = new DataPlane(db, resolver.baseResolver);
53
+
const dataplane = new DataPlane(db, idResolver);
175
54
const hydrator = new Hydrator(dataplane);
176
-
const views = new Views();
55
+
const views = new Views({
56
+
indexedAtEpoch: cfg.indexedAtEpoch,
57
+
videoCdn: cfg.videoCdn,
58
+
hlsCdn: cfg.hlsCdn,
59
+
mediaCdn: cfg.mediaCdn,
60
+
thumbCdn: cfg.thumbCdn,
61
+
});
177
62
178
-
// Services
179
-
const sub = new RepoSubscription({
180
-
service: env.RELAY_URL,
181
-
db,
182
-
idResolver: baseIdResolver,
183
-
startCursor,
184
-
});
185
-
const takedownService = new TakedownService(db);
186
63
const authVerifier = createAuthVerifier(dataplane, {
187
-
ownDid: serviceDid,
64
+
ownDid: cfg.serverDid,
188
65
alternateAudienceDids: [],
189
-
modServiceDid: env.MOD_SERVICE_DID,
190
-
adminPasses: [env.ADMIN_PASSWORD],
66
+
modServiceDid: cfg.modServiceDid,
67
+
adminPasses: cfg.adminPasswords,
191
68
});
192
69
193
70
const ctx = {
···
196
73
hydrator,
197
74
views,
198
75
logger: appLogger,
199
-
resolver,
200
-
serviceDid,
201
-
didResolver: baseIdResolver.did,
202
-
takedownService,
76
+
idResolver,
77
+
cfg,
203
78
authVerifier,
204
-
sub,
205
79
};
206
80
207
81
const app = createApp(ctx);
···
209
83
}
210
84
211
85
// Start server function
212
-
export async function startServer() {
213
-
const { app, ctx } = await setupApp();
86
+
export function startServer() {
87
+
const { app, ctx } = setupApp();
214
88
215
89
// Start HTTP server immediately
216
-
const { HOST, PORT } = env;
90
+
const { port } = ctx.cfg;
217
91
Deno.serve({
218
-
hostname: HOST,
219
-
port: PORT,
92
+
port,
220
93
onListen: (info) => {
221
94
ctx.logger.info(`Server listening on ${info.hostname}:${info.port}`);
222
95
},
223
96
}, app.fetch);
224
97
225
-
// Start subscription only after DB is connected
226
-
let stopStartLoop = false;
227
-
let retryTimeoutId: number | undefined;
228
-
let retryResolve: (() => void) | null = null;
229
-
const startSubWhenReady = async () => {
230
-
ctx.logger.info(
231
-
"Waiting for MongoDB connection before starting subscription...",
232
-
);
233
-
let attempt = 0;
234
-
while (!stopStartLoop) {
235
-
attempt++;
236
-
const connected = await ctx.db.waitForConnection(30000);
237
-
if (connected) {
238
-
ctx.logger.info("MongoDB connected; starting firehose subscription");
239
-
ctx.sub.start();
240
-
break;
241
-
} else {
242
-
ctx.logger.error(
243
-
`MongoDB not connected after timeout (attempt ${attempt}); retrying in 5s...`,
244
-
);
245
-
await new Promise<void>((resolve) => {
246
-
retryResolve = () => {
247
-
resolve();
248
-
retryResolve = null;
249
-
};
250
-
retryTimeoutId = setTimeout(() => {
251
-
retryResolve = null;
252
-
resolve();
253
-
}, 5000);
254
-
});
255
-
retryTimeoutId = undefined;
256
-
}
257
-
}
258
-
};
259
-
startSubWhenReady(); // fire and forget
260
-
261
98
// Handle shutdown
262
99
const shutdown = async (signal: string) => {
263
100
ctx.logger.info(`Received ${signal}; shutting down...`);
264
-
stopStartLoop = true;
265
-
if (retryTimeoutId !== undefined) {
266
-
clearTimeout(retryTimeoutId);
267
-
retryTimeoutId = undefined;
268
-
}
269
-
if (retryResolve) {
270
-
retryResolve();
271
-
retryResolve = null;
272
-
}
273
101
try {
274
-
await ctx.sub.destroy();
275
-
} catch (e) {
276
-
ctx.logger.error("Error destroying subscription during shutdown", { e });
277
-
}
278
-
try {
102
+
ctx.logger.info("Disconnecting database...");
279
103
await ctx.db.disconnect();
280
-
} catch (e) {
281
-
ctx.logger.error("Error disconnecting database during shutdown", { e });
104
+
} catch (err) {
105
+
ctx.logger.error("Error disconnecting database during shutdown", { err });
282
106
}
283
107
ctx.logger.info("Shutdown complete");
284
108
Deno.exit(0);
···
288
112
Deno.addSignalListener("SIGTERM", () => shutdown("SIGTERM"));
289
113
}
290
114
291
-
// Default export for backward compatibility (creates app without starting services)
292
-
let defaultApp: Hono<AppEnv> | null = null;
293
-
294
-
export default async function getApp(): Promise<Hono<AppEnv>> {
295
-
if (!defaultApp) {
296
-
const result = await setupApp();
297
-
defaultApp = result.app;
298
-
}
299
-
return defaultApp;
300
-
}
301
-
302
115
// Start the server if this file is run directly
303
116
if (import.meta.main) {
304
-
await startServer();
117
+
startServer();
305
118
}
-219
main_test.ts
-219
main_test.ts
···
1
-
import { assertEquals } from "@std/assert";
2
-
import { assertMatch } from "@std/assert/match";
3
-
import { AppContext, createApp } from "./main.ts";
4
-
import { Database } from "./data-plane/db/index.ts";
5
-
import {
6
-
createBidirectionalResolver,
7
-
createIdResolver,
8
-
} from "./utils/id-resolver.ts";
9
-
import { TakedownService } from "./services/takedown.ts";
10
-
import { createAuthVerifier } from "./auth-verifier.ts";
11
-
import { RepoSubscription } from "./data-plane/subscription.ts";
12
-
import { MemoryRunner } from "@atp/sync";
13
-
import { getLogger } from "@logtape/logtape";
14
-
import { DataPlane } from "./data-plane/index.ts";
15
-
import { Hydrator } from "./hydration/index.ts";
16
-
import { Views } from "./views/index.ts";
17
-
18
-
Deno.env.set("SERVICE_DID", "did:web:test");
19
-
Deno.env.set("MOD_SERVICE_DID", "did:web:test");
20
-
Deno.env.set("ADMIN_PASSWORD", "test");
21
-
Deno.env.set(
22
-
"APPVIEW_K256_PRIVATE_KEY_HEX",
23
-
"5676df35fd3a185a1771a43536635ad90057e0c0d1fd91436344bb50ce23a460", // random valid test key
24
-
);
25
-
26
-
// Create a mock context for testing without database
27
-
function createMockContext(): AppContext {
28
-
const appLogger = getLogger(["appview"]);
29
-
const baseIdResolver = createIdResolver();
30
-
const resolver = createBidirectionalResolver(baseIdResolver);
31
-
const serviceDid = "did:web:test";
32
-
33
-
// Create mock database that doesn't actually connect
34
-
const mockDb = {
35
-
connect: () => Promise.resolve(),
36
-
disconnect: () => Promise.resolve(),
37
-
models: {},
38
-
getCursorState: () => Promise.resolve(null),
39
-
saveCursorState: () => Promise.resolve(),
40
-
} as unknown as Database;
41
-
42
-
const dataplane = new DataPlane(mockDb, resolver.baseResolver);
43
-
const hydrator = new Hydrator(dataplane);
44
-
const views = new Views();
45
-
const takedownService = new TakedownService(mockDb);
46
-
const sub = new RepoSubscription({
47
-
service: "wss://relay1.us-west.bsky.network",
48
-
db: mockDb,
49
-
idResolver: baseIdResolver,
50
-
startCursor: undefined,
51
-
});
52
-
const authVerifier = createAuthVerifier(dataplane, {
53
-
ownDid: serviceDid,
54
-
alternateAudienceDids: [],
55
-
modServiceDid: "did:web:test",
56
-
adminPasses: ["test"],
57
-
});
58
-
59
-
return {
60
-
db: mockDb,
61
-
dataplane,
62
-
hydrator,
63
-
views,
64
-
logger: appLogger,
65
-
resolver,
66
-
serviceDid,
67
-
didResolver: baseIdResolver.did,
68
-
takedownService,
69
-
sub,
70
-
authVerifier,
71
-
};
72
-
}
73
-
74
-
Deno.test("Basic App Creation", async () => {
75
-
console.log("Testing basic app creation...");
76
-
77
-
const ctx = createMockContext();
78
-
const app = createApp(ctx);
79
-
80
-
console.log("App created successfully");
81
-
82
-
const res = await app.request("/", {
83
-
headers: {
84
-
"Content-Type": "application/json",
85
-
},
86
-
});
87
-
88
-
assertEquals(res.status, 200);
89
-
console.log("Basic app test passed");
90
-
});
91
-
92
-
Deno.test("Well Known Endpoint", async () => {
93
-
console.log("Testing well-known endpoint...");
94
-
95
-
const ctx = createMockContext();
96
-
const app = createApp(ctx);
97
-
98
-
const res = await app.request("/.well-known/did.json", {
99
-
headers: {
100
-
"Content-Type": "application/json",
101
-
},
102
-
});
103
-
104
-
assertMatch(
105
-
await res.text(),
106
-
new RegExp(
107
-
[
108
-
"^\\{",
109
-
'"@context":\\["https://www\\.w3\\.org/ns/did/v1","https://w3id\\.org/security/multikey/v1"\\],',
110
-
'"id":"(did:web:[^"]+)",',
111
-
'"verificationMethod":\\[\\{',
112
-
'"id":"\\1#atproto",',
113
-
'"type":"Multikey",',
114
-
'"controller":"\\1",',
115
-
'"publicKeyMultibase":"[a-zA-Z0-9]+"',
116
-
"\\}\\],",
117
-
'"service":\\[\\{',
118
-
'"id":"#sprk_appview",',
119
-
'"type":"SprkAppView",',
120
-
'"serviceEndpoint":"https?://[^"]+"',
121
-
"\\}\\]",
122
-
"\\}$",
123
-
].join(""),
124
-
),
125
-
);
126
-
console.log("Well-known endpoint test passed");
127
-
});
128
-
129
-
Deno.test("Cursor Persistence Test", async () => {
130
-
console.log("Testing cursor persistence...");
131
-
132
-
// Mock database with cursor state
133
-
let storedCursor: number | null = 42;
134
-
const mockDb = {
135
-
connect: () => Promise.resolve(),
136
-
disconnect: () => Promise.resolve(),
137
-
waitForConnection: () => Promise.resolve(true),
138
-
models: {},
139
-
getCursorState: () => Promise.resolve(storedCursor),
140
-
saveCursorState: (cursor: number) => {
141
-
storedCursor = cursor;
142
-
return Promise.resolve();
143
-
},
144
-
} as unknown as Database;
145
-
146
-
const baseIdResolver = createIdResolver();
147
-
148
-
// Test 1: Cursor is loaded from database
149
-
const sub1 = new RepoSubscription({
150
-
service: "wss://relay1.us-west.bsky.network",
151
-
db: mockDb,
152
-
idResolver: baseIdResolver,
153
-
startCursor: 42, // This should be what was read from DB
154
-
});
155
-
156
-
console.log("Initial cursor:", sub1.runner.getCursor());
157
-
158
-
// Test 2: Simulate cursor update
159
-
if (sub1.runner.opts.setCursor) {
160
-
await sub1.runner.opts.setCursor(100);
161
-
}
162
-
163
-
console.log("Stored cursor after update:", storedCursor);
164
-
165
-
// Test 3: Create new subscription, should load updated cursor
166
-
const sub2 = new RepoSubscription({
167
-
service: "wss://relay1.us-west.bsky.network",
168
-
db: mockDb,
169
-
idResolver: baseIdResolver,
170
-
startCursor: 100, // This should be the updated cursor from DB
171
-
});
172
-
173
-
console.log("New subscription cursor:", sub2.runner.getCursor());
174
-
175
-
console.log("Cursor persistence test passed");
176
-
});
177
-
178
-
Deno.test("Cursor Save Throttling Test", async () => {
179
-
console.log("Testing cursor save throttling...");
180
-
181
-
let saveCount = 0;
182
-
let lastSavedCursor: number | undefined;
183
-
184
-
// Create a direct MemoryRunner to test throttling
185
-
const runner = new MemoryRunner({
186
-
startCursor: 0,
187
-
setCursorInterval: 100, // Use 100ms for faster testing
188
-
setCursor: (cursor: number): Promise<void> => {
189
-
saveCount++;
190
-
lastSavedCursor = cursor;
191
-
console.log(`Save #${saveCount}: cursor ${cursor}`);
192
-
return Promise.resolve();
193
-
},
194
-
});
195
-
196
-
// Simulate rapid cursor updates through trackEvent (the proper way)
197
-
await runner.trackEvent("did1", 10, async () => {/* mock work */});
198
-
await runner.trackEvent("did2", 20, async () => {/* mock work */});
199
-
await runner.trackEvent("did3", 30, async () => {/* mock work */});
200
-
await runner.trackEvent("did4", 40, async () => {/* mock work */});
201
-
await runner.trackEvent("did5", 50, async () => {/* mock work */});
202
-
203
-
console.log(`Immediate saves: ${saveCount}`);
204
-
console.log(`Last saved cursor: ${lastSavedCursor}`);
205
-
206
-
// Wait for throttling to potentially trigger more saves
207
-
await new Promise((resolve) => setTimeout(resolve, 200));
208
-
209
-
console.log(`After delay - Save count: ${saveCount}`);
210
-
console.log(`Last saved cursor: ${lastSavedCursor}`);
211
-
212
-
// Force save on destroy
213
-
await runner.destroy();
214
-
215
-
console.log(`After destroy - Save count: ${saveCount}`);
216
-
console.log(`Final saved cursor: ${lastSavedCursor}`);
217
-
218
-
console.log("Cursor save throttling test completed");
219
-
});
-383
services/takedown-filter.ts
-383
services/takedown-filter.ts
···
1
-
import { Context, Next } from "hono";
2
-
import { TakedownService } from "./takedown.ts";
3
-
import * as SoSprkFeedDefs from "../lex/types/so/sprk/feed/defs.ts";
4
-
5
-
/**
6
-
* Middleware that filters out taken-down content from responses
7
-
* This is meant to be applied to routes that return content
8
-
* that might have been taken down by admins
9
-
*/
10
-
export const takedownFilterMiddleware = async (c: Context, next: Next) => {
11
-
// Skip filtering if user is an admin
12
-
const isAdmin = c.get("isAdmin") as boolean | undefined;
13
-
if (isAdmin) {
14
-
await next();
15
-
return;
16
-
}
17
-
18
-
await next();
19
-
20
-
const contentType = c.res.headers.get("Content-Type");
21
-
if (!contentType || !contentType.includes("application/json")) {
22
-
return;
23
-
}
24
-
25
-
try {
26
-
// Get the takedown service from context
27
-
const takedownService = c.env.takedownService as TakedownService;
28
-
29
-
const body = await c.res.json();
30
-
31
-
const targetDid = body.did ||
32
-
body.user?.did ||
33
-
body.actor?.did ||
34
-
body.profile?.did ||
35
-
body.subject?.did;
36
-
if (targetDid) {
37
-
const repoTakedown = await takedownService.getRepoTakedown(targetDid);
38
-
if (repoTakedown?.applied) {
39
-
// For specific user/profile views, return minimal placeholder
40
-
if (body.did && body.$type && body.$type.includes("profileView")) {
41
-
const takenDownProfile = {
42
-
$type: body.$type,
43
-
did: body.did,
44
-
handle: body.handle || "unavailable",
45
-
moderation: {
46
-
takenDown: true,
47
-
},
48
-
};
49
-
c.res = new Response(JSON.stringify(takenDownProfile), {
50
-
status: c.res.status,
51
-
headers: c.res.headers,
52
-
});
53
-
return;
54
-
} else {
55
-
// For other single-user responses, null out or minimize the content
56
-
c.res = new Response(
57
-
JSON.stringify({
58
-
error: "Content unavailable - repository has been taken down",
59
-
code: 404,
60
-
}),
61
-
{
62
-
status: 404,
63
-
headers: c.res.headers,
64
-
},
65
-
);
66
-
return;
67
-
}
68
-
}
69
-
}
70
-
71
-
// Continue with specific content type filtering
72
-
if (body.posts && Array.isArray(body.posts)) {
73
-
const filteredPosts = await filterTakenDownItems(
74
-
body.posts,
75
-
takedownService,
76
-
"uri",
77
-
);
78
-
body.posts = filteredPosts;
79
-
} else if (body.feed && Array.isArray(body.feed)) {
80
-
const filteredFeed = await filterTakenDownItems(
81
-
body.feed,
82
-
takedownService,
83
-
"post.uri",
84
-
);
85
-
body.feed = filteredFeed;
86
-
} else if (body.thread && body.thread.post) {
87
-
const takedown = await takedownService.getTakedown(body.thread.post.uri);
88
-
const isThreadTakenDown = takedown?.applied ?? false;
89
-
90
-
// Also check if the thread author repo is taken down
91
-
let isAuthorTakenDown = false;
92
-
if (body.thread.post.author?.did) {
93
-
const repoTakedown = await takedownService.getRepoTakedown(
94
-
body.thread.post.author.did,
95
-
);
96
-
isAuthorTakenDown = repoTakedown?.applied ?? false;
97
-
}
98
-
99
-
if (isThreadTakenDown || isAuthorTakenDown) {
100
-
body.thread = null;
101
-
} else if (body.thread.replies) {
102
-
body.thread.replies = await filterReplies(
103
-
body.thread.replies,
104
-
takedownService,
105
-
);
106
-
}
107
-
}
108
-
109
-
// If there are user profiles in the response, filter out taken down repositories
110
-
if (body.profiles && Array.isArray(body.profiles)) {
111
-
const filteredProfiles = await filterTakenDownRepos(
112
-
body.profiles,
113
-
takedownService,
114
-
);
115
-
body.profiles = filteredProfiles;
116
-
} else if (body.profile) {
117
-
if (body.profile.did) {
118
-
const repoTakedown = await takedownService.getRepoTakedown(
119
-
body.profile.did,
120
-
);
121
-
if (repoTakedown?.applied) {
122
-
body.profile = null;
123
-
}
124
-
}
125
-
} else if (body.did && body.$type && body.$type.includes("profileView")) {
126
-
// For direct ProfileViewDetailed objects (so.sprk.actor.getProfile)
127
-
const repoTakedown = await takedownService.getRepoTakedown(body.did);
128
-
if (repoTakedown?.applied) {
129
-
// Return a minimal placeholder object for taken-down profiles
130
-
const takenDownProfile = {
131
-
$type: body.$type,
132
-
did: body.did,
133
-
handle: body.handle || "unavailable",
134
-
moderation: {
135
-
takenDown: true,
136
-
},
137
-
};
138
-
139
-
// Create a new response with the placeholder instead of trying to modify body
140
-
c.res = new Response(JSON.stringify(takenDownProfile), {
141
-
status: c.res.status,
142
-
headers: c.res.headers,
143
-
});
144
-
145
-
// Skip the rest of the processing
146
-
return;
147
-
}
148
-
} else if (body.subject) {
149
-
// For followers/follows response that has a subject profile
150
-
if (body.subject.did) {
151
-
const repoTakedown = await takedownService.getRepoTakedown(
152
-
body.subject.did,
153
-
);
154
-
if (repoTakedown?.applied) {
155
-
// Keep minimal info about the profile but mark it as taken down
156
-
body.subject = {
157
-
$type: body.subject.$type,
158
-
did: body.subject.did,
159
-
handle: body.subject.handle || "unavailable",
160
-
moderation: {
161
-
takenDown: true,
162
-
},
163
-
};
164
-
}
165
-
}
166
-
167
-
// Also filter any followers/follows list
168
-
if (body.followers && Array.isArray(body.followers)) {
169
-
body.followers = await filterTakenDownRepos(
170
-
body.followers,
171
-
takedownService,
172
-
);
173
-
}
174
-
175
-
if (body.follows && Array.isArray(body.follows)) {
176
-
body.follows = await filterTakenDownRepos(
177
-
body.follows,
178
-
takedownService,
179
-
);
180
-
}
181
-
}
182
-
183
-
// Set the filtered response
184
-
c.res = new Response(JSON.stringify(body), {
185
-
status: c.res.status,
186
-
headers: c.res.headers,
187
-
});
188
-
} catch (error) {
189
-
// In case of error, just continue with the original response
190
-
console.error("Error in takedown filter middleware:", error);
191
-
}
192
-
};
193
-
194
-
// Helper function to filter out taken down items
195
-
async function filterTakenDownItems(
196
-
items: (SoSprkFeedDefs.PostView | SoSprkFeedDefs.FeedViewPost)[],
197
-
takedownService: TakedownService,
198
-
uriPath: string,
199
-
) {
200
-
if (!items || items.length === 0) {
201
-
return items;
202
-
}
203
-
204
-
const filteredItems:
205
-
(SoSprkFeedDefs.PostView | SoSprkFeedDefs.FeedViewPost)[] = [];
206
-
207
-
for (const item of items) {
208
-
let isTakenDown = false;
209
-
210
-
// Get URI for this specific content
211
-
const uri = getNestedProperty(item, uriPath);
212
-
if (uri && typeof uri === "string") {
213
-
const takedown = await takedownService.getTakedown(uri);
214
-
isTakenDown = takedown?.applied ?? false;
215
-
}
216
-
217
-
// Check if author's repo is taken down
218
-
let isAuthorTakenDown = false;
219
-
// Look for author DID in common locations
220
-
const authorDid = getNestedProperty(item, "author.did") ||
221
-
getNestedProperty(item, "post.author.did") ||
222
-
getNestedProperty(item, "user.did") ||
223
-
getNestedProperty(item, "actor.did");
224
-
225
-
if (authorDid && typeof authorDid === "string") {
226
-
const repoTakedown = await takedownService.getRepoTakedown(authorDid);
227
-
isAuthorTakenDown = repoTakedown?.applied ?? false;
228
-
}
229
-
230
-
// Keep the item only if neither the content nor the author is taken down
231
-
if (!isTakenDown && !isAuthorTakenDown) {
232
-
filteredItems.push(item);
233
-
}
234
-
}
235
-
236
-
return filteredItems;
237
-
}
238
-
239
-
// Helper function to filter out taken down repositories
240
-
async function filterTakenDownRepos(
241
-
profiles: { $type: string; did: string; handle?: string }[],
242
-
takedownService: TakedownService,
243
-
): Promise<
244
-
{
245
-
$type: string;
246
-
did: string;
247
-
handle?: string;
248
-
moderation?: { takenDown: boolean };
249
-
}[]
250
-
> {
251
-
if (!profiles || !Array.isArray(profiles)) return profiles;
252
-
253
-
const filteredProfiles: {
254
-
$type: string;
255
-
did: string;
256
-
handle?: string;
257
-
moderation?: { takenDown: boolean };
258
-
}[] = [];
259
-
260
-
for (const profile of profiles) {
261
-
if (profile.did) {
262
-
const repoTakedown = await takedownService.getRepoTakedown(profile.did);
263
-
if (!repoTakedown?.applied) {
264
-
filteredProfiles.push(profile);
265
-
} else {
266
-
// For UI consistency, push a minimal placeholder for taken-down profiles
267
-
// if they need to be represented in lists (follows, followers, etc.)
268
-
filteredProfiles.push({
269
-
$type: profile.$type,
270
-
did: profile.did,
271
-
handle: profile.handle || "unavailable",
272
-
moderation: {
273
-
takenDown: true,
274
-
},
275
-
});
276
-
}
277
-
} else {
278
-
// If no DID, keep the profile
279
-
filteredProfiles.push(profile);
280
-
}
281
-
}
282
-
283
-
return filteredProfiles;
284
-
}
285
-
286
-
// Helper function to filter out taken down blobs/images
287
-
async function filterTakenDownBlobs(
288
-
images: { cid?: string; did?: string }[],
289
-
takedownService: TakedownService,
290
-
): Promise<{ cid?: string; did?: string }[]> {
291
-
if (!images || !Array.isArray(images)) return images;
292
-
293
-
const filteredImages: { cid?: string; did?: string }[] = [];
294
-
295
-
for (const image of images) {
296
-
// Check if the image is taken down based on blob CID
297
-
if (image.cid && image.did) {
298
-
const blobTakedown = await takedownService.getBlobTakedown(
299
-
image.did,
300
-
image.cid,
301
-
);
302
-
if (!blobTakedown?.applied) {
303
-
filteredImages.push(image);
304
-
}
305
-
} else {
306
-
// If no CID or DID, keep the image
307
-
filteredImages.push(image);
308
-
}
309
-
}
310
-
311
-
return filteredImages;
312
-
}
313
-
314
-
type ReplyType = {
315
-
post?: {
316
-
uri: string;
317
-
author?: { did: string };
318
-
embed?: { images?: { cid?: string; did?: string }[] };
319
-
};
320
-
replies?: ReplyType[];
321
-
};
322
-
323
-
async function filterReplies(
324
-
replies: ReplyType[],
325
-
takedownService: TakedownService,
326
-
): Promise<ReplyType[]> {
327
-
if (!replies || !Array.isArray(replies)) return replies;
328
-
329
-
const filteredReplies: ReplyType[] = [];
330
-
331
-
for (const reply of replies) {
332
-
if (reply.post && reply.post.uri) {
333
-
const takedown = await takedownService.getTakedown(reply.post.uri);
334
-
const isTakenDown = takedown?.applied ?? false;
335
-
336
-
// Check if author's repo is taken down
337
-
let isAuthorTakenDown = false;
338
-
if (reply.post.author?.did) {
339
-
const repoTakedown = await takedownService.getRepoTakedown(
340
-
reply.post.author.did,
341
-
);
342
-
isAuthorTakenDown = repoTakedown?.applied ?? false;
343
-
}
344
-
345
-
if (!isTakenDown && !isAuthorTakenDown) {
346
-
// If this reply has nested replies, filter those too
347
-
if (reply.replies && Array.isArray(reply.replies)) {
348
-
reply.replies = await filterReplies(reply.replies, takedownService);
349
-
}
350
-
351
-
// Filter out taken down images in the post
352
-
if (
353
-
reply.post.embed?.images &&
354
-
Array.isArray(reply.post.embed.images)
355
-
) {
356
-
reply.post.embed.images = await filterTakenDownBlobs(
357
-
reply.post.embed.images,
358
-
takedownService,
359
-
);
360
-
}
361
-
362
-
filteredReplies.push(reply);
363
-
}
364
-
} else {
365
-
// If no post or URI, keep the reply
366
-
filteredReplies.push(reply);
367
-
}
368
-
}
369
-
370
-
return filteredReplies;
371
-
}
372
-
373
-
// Helper function to safely access nested object properties
374
-
function getNestedProperty(obj: unknown, path: string): unknown {
375
-
if (!obj || typeof obj !== "object") return undefined;
376
-
377
-
return path.split(".").reduce((current: unknown, key: string): unknown => {
378
-
return current && typeof current === "object" && current !== null &&
379
-
key in current
380
-
? (current as Record<string, unknown>)[key]
381
-
: undefined;
382
-
}, obj);
383
-
}
-371
services/takedown.ts
-371
services/takedown.ts
···
1
-
import {
2
-
BlobTakedownDocument,
3
-
RepoTakedownDocument,
4
-
TakedownDocument,
5
-
} from "../data-plane/db/models.ts";
6
-
import { Database } from "../data-plane/db/index.ts";
7
-
8
-
export class TakedownService {
9
-
constructor(private db: Database) {}
10
-
11
-
async takedownContent(params: {
12
-
targetUri: string;
13
-
targetCid: string;
14
-
reason: string;
15
-
adminDid: string;
16
-
ref?: string;
17
-
}): Promise<void> {
18
-
const { targetUri, targetCid, reason, adminDid, ref } = params;
19
-
20
-
// Create a takedown record
21
-
await this.db.models.Takedown.create({
22
-
targetUri,
23
-
targetCid,
24
-
reason,
25
-
takenDownBy: adminDid,
26
-
takenDownAt: new Date().toISOString(),
27
-
ref: ref || null,
28
-
applied: true,
29
-
});
30
-
31
-
// Update the record document with takedown status
32
-
await this.updateRecordTakedownStatus(
33
-
targetUri,
34
-
true,
35
-
ref || "BSKY-TAKEDOWN-UNKNOWN",
36
-
);
37
-
}
38
-
39
-
// Add a method to handle user repo takedowns
40
-
async takedownRepo(params: {
41
-
did: string;
42
-
reason: string;
43
-
adminDid: string;
44
-
ref?: string;
45
-
}): Promise<void> {
46
-
const { did, reason, adminDid, ref } = params;
47
-
48
-
// Create a repo takedown record
49
-
await this.db.models.RepoTakedown.create({
50
-
did,
51
-
reason,
52
-
takenDownBy: adminDid,
53
-
takenDownAt: new Date().toISOString(),
54
-
ref: ref || null,
55
-
applied: false,
56
-
});
57
-
58
-
// Update all records from this DID with takedown status
59
-
await this.updateAllRecordsTakedownStatusByDid(
60
-
did,
61
-
true,
62
-
ref || "BSKY-TAKEDOWN-UNKNOWN",
63
-
);
64
-
}
65
-
66
-
// Add a method to handle blob takedowns
67
-
async takedownBlob(params: {
68
-
did: string;
69
-
cid: string;
70
-
reason: string;
71
-
adminDid: string;
72
-
ref?: string;
73
-
}): Promise<void> {
74
-
const { did, cid, reason, adminDid, ref } = params;
75
-
76
-
// Create a blob takedown record
77
-
await this.db.models.BlobTakedown.create({
78
-
did,
79
-
cid,
80
-
reason,
81
-
takenDownBy: adminDid,
82
-
takenDownAt: new Date().toISOString(),
83
-
ref: ref || null,
84
-
applied: false,
85
-
});
86
-
}
87
-
88
-
async isTakenDown(uri: string): Promise<boolean> {
89
-
const takedown = await this.db.models.Takedown.findOne({ targetUri: uri });
90
-
return !!takedown;
91
-
}
92
-
93
-
/**
94
-
* Get takedown information for a URI if it exists
95
-
* @param uri The URI of the content to check
96
-
* @returns Takedown information or null if not taken down
97
-
*/
98
-
async getTakedown(uri: string): Promise<
99
-
{
100
-
targetUri: string;
101
-
targetCid: string;
102
-
reason: string;
103
-
takenDownBy: string;
104
-
takenDownAt: string;
105
-
applied: boolean;
106
-
} | null
107
-
> {
108
-
const takedown = await this.db.models.Takedown.findOne({ targetUri: uri })
109
-
.lean();
110
-
return takedown;
111
-
}
112
-
113
-
// Add a method to check if a repo is taken down
114
-
async isRepoTakenDown(did: string): Promise<boolean> {
115
-
const takedown = await this.db.models.RepoTakedown.findOne({ did });
116
-
return !!takedown;
117
-
}
118
-
119
-
// Add a method to check if a blob is taken down
120
-
async isBlobTakenDown(did: string, cid: string): Promise<boolean> {
121
-
const takedown = await this.db.models.BlobTakedown.findOne({ did, cid });
122
-
return !!takedown;
123
-
}
124
-
125
-
async removeTakedown(targetUri: string): Promise<boolean> {
126
-
const result = await this.db.models.Takedown.deleteOne({ targetUri });
127
-
128
-
// Update the record document to remove takedown status
129
-
if (result.deletedCount > 0) {
130
-
await this.updateRecordTakedownStatus(targetUri, false);
131
-
}
132
-
133
-
return result.deletedCount > 0;
134
-
}
135
-
136
-
// Add a method to remove repo takedown
137
-
async removeRepoTakedown(did: string): Promise<boolean> {
138
-
const result = await this.db.models.RepoTakedown.deleteOne({ did });
139
-
140
-
// Update all records from this DID to remove takedown status
141
-
if (result.deletedCount > 0) {
142
-
await this.updateAllRecordsTakedownStatusByDid(did, false);
143
-
}
144
-
145
-
return result.deletedCount > 0;
146
-
}
147
-
148
-
// Add a method to remove blob takedown
149
-
async removeBlobTakedown(did: string, cid: string): Promise<boolean> {
150
-
const result = await this.db.models.BlobTakedown.deleteOne({ did, cid });
151
-
return result.deletedCount > 0;
152
-
}
153
-
154
-
async listTakedowns(limit: number = 50, cursor?: string): Promise<{
155
-
takedowns: Array<{
156
-
targetUri: string;
157
-
targetCid: string;
158
-
reason: string;
159
-
takenDownBy: string;
160
-
takenDownAt: string;
161
-
}>;
162
-
cursor?: string;
163
-
}> {
164
-
const query = cursor ? { targetUri: { $lt: cursor } } : {};
165
-
166
-
const takedowns = await this.db.models.Takedown
167
-
.find(query)
168
-
.sort({ targetUri: -1 })
169
-
.limit(limit + 1);
170
-
171
-
const items = takedowns.slice(0, limit);
172
-
173
-
return {
174
-
takedowns: items.map((t: TakedownDocument) => ({
175
-
targetUri: t.targetUri,
176
-
targetCid: t.targetCid,
177
-
reason: t.reason,
178
-
takenDownBy: t.takenDownBy,
179
-
takenDownAt: t.takenDownAt,
180
-
})),
181
-
cursor: takedowns.length > limit
182
-
? takedowns[limit - 1].targetUri
183
-
: undefined,
184
-
};
185
-
}
186
-
187
-
// Add a method to list repo takedowns
188
-
async listRepoTakedowns(limit: number = 50, cursor?: string): Promise<{
189
-
repoTakedowns: Array<{
190
-
did: string;
191
-
reason: string;
192
-
takenDownBy: string;
193
-
takenDownAt: string;
194
-
ref: string | null;
195
-
}>;
196
-
cursor?: string;
197
-
}> {
198
-
const query = cursor ? { did: { $lt: cursor } } : {};
199
-
200
-
const takedowns = await this.db.models.RepoTakedown
201
-
.find(query)
202
-
.sort({ did: -1 })
203
-
.limit(limit + 1);
204
-
205
-
const items = takedowns.slice(0, limit);
206
-
207
-
return {
208
-
repoTakedowns: items.map((t: RepoTakedownDocument) => ({
209
-
did: t.did,
210
-
reason: t.reason,
211
-
takenDownBy: t.takenDownBy,
212
-
takenDownAt: t.takenDownAt,
213
-
ref: t.ref,
214
-
})),
215
-
cursor: takedowns.length > limit ? takedowns[limit - 1].did : undefined,
216
-
};
217
-
}
218
-
219
-
// Add a method to list blob takedowns
220
-
async listBlobTakedowns(limit: number = 50, cursor?: string): Promise<{
221
-
blobTakedowns: Array<{
222
-
did: string;
223
-
cid: string;
224
-
reason: string;
225
-
takenDownBy: string;
226
-
takenDownAt: string;
227
-
ref: string | null;
228
-
}>;
229
-
cursor?: string;
230
-
}> {
231
-
const query = cursor ? { did: { $lt: cursor } } : {};
232
-
233
-
const takedowns = await this.db.models.BlobTakedown
234
-
.find(query)
235
-
.sort({ did: -1, cid: -1 })
236
-
.limit(limit + 1);
237
-
238
-
const items = takedowns.slice(0, limit);
239
-
240
-
return {
241
-
blobTakedowns: items.map((t: BlobTakedownDocument) => ({
242
-
did: t.did,
243
-
cid: t.cid,
244
-
reason: t.reason,
245
-
takenDownBy: t.takenDownBy,
246
-
takenDownAt: t.takenDownAt,
247
-
ref: t.ref,
248
-
})),
249
-
cursor: takedowns.length > limit ? takedowns[limit - 1].did : undefined,
250
-
};
251
-
}
252
-
253
-
async updateTakedownApplied(
254
-
targetUri: string,
255
-
applied: boolean,
256
-
): Promise<void> {
257
-
await this.db.models.Takedown.updateOne(
258
-
{ targetUri },
259
-
{ $set: { applied } },
260
-
);
261
-
}
262
-
263
-
async updateRepoTakedownApplied(
264
-
did: string,
265
-
applied: boolean,
266
-
): Promise<void> {
267
-
await this.db.models.RepoTakedown.updateOne(
268
-
{ did },
269
-
{ $set: { applied } },
270
-
);
271
-
}
272
-
273
-
async updateBlobTakedownApplied(
274
-
did: string,
275
-
cid: string,
276
-
applied: boolean,
277
-
): Promise<void> {
278
-
await this.db.models.BlobTakedown.updateOne(
279
-
{ did, cid },
280
-
{ $set: { applied } },
281
-
);
282
-
}
283
-
284
-
async getRepoTakedown(did: string): Promise<
285
-
{
286
-
did: string;
287
-
reason: string;
288
-
takenDownBy: string;
289
-
takenDownAt: string;
290
-
ref: string | null;
291
-
applied: boolean;
292
-
} | null
293
-
> {
294
-
const takedown = await this.db.models.RepoTakedown.findOne({ did }).lean();
295
-
return takedown;
296
-
}
297
-
298
-
async getBlobTakedown(did: string, cid: string): Promise<
299
-
{
300
-
did: string;
301
-
cid: string;
302
-
reason: string;
303
-
takenDownBy: string;
304
-
takenDownAt: string;
305
-
ref: string | null;
306
-
applied: boolean;
307
-
} | null
308
-
> {
309
-
const takedown = await this.db.models.BlobTakedown.findOne({ did, cid })
310
-
.lean();
311
-
return takedown;
312
-
}
313
-
314
-
/**
315
-
* Update the takenDown and takedownRef properties on a RecordDocument
316
-
* @param uri The URI of the record to update
317
-
* @param takenDown Whether the record is taken down
318
-
* @param takedownRef Optional reference for the takedown
319
-
*/
320
-
async updateRecordTakedownStatus(
321
-
uri: string,
322
-
takenDown: boolean,
323
-
takedownRef?: string,
324
-
): Promise<void> {
325
-
const updateData: { takenDown: boolean; takedownRef?: string } = {
326
-
takenDown,
327
-
};
328
-
329
-
if (takenDown && takedownRef) {
330
-
updateData.takedownRef = takedownRef;
331
-
await this.db.models.Record.updateOne(
332
-
{ uri },
333
-
{ $set: updateData },
334
-
);
335
-
} else if (!takenDown) {
336
-
await this.db.models.Record.updateOne(
337
-
{ uri },
338
-
{ $set: { takenDown }, $unset: { takedownRef: "" } },
339
-
);
340
-
}
341
-
}
342
-
343
-
/**
344
-
* Update the takenDown and takedownRef properties on all RecordDocuments for a specific DID
345
-
* @param did The DID of the user whose records should be updated
346
-
* @param takenDown Whether the records are taken down
347
-
* @param takedownRef Optional reference for the takedown
348
-
*/
349
-
async updateAllRecordsTakedownStatusByDid(
350
-
did: string,
351
-
takenDown: boolean,
352
-
takedownRef?: string,
353
-
): Promise<void> {
354
-
if (takenDown && takedownRef) {
355
-
await this.db.models.Record.updateMany(
356
-
{ did },
357
-
{ $set: { takenDown, takedownRef } },
358
-
);
359
-
} else if (!takenDown) {
360
-
await this.db.models.Record.updateMany(
361
-
{ did },
362
-
{ $set: { takenDown }, $unset: { takedownRef: "" } },
363
-
);
364
-
} else {
365
-
await this.db.models.Record.updateMany(
366
-
{ did },
367
-
{ $set: { takenDown } },
368
-
);
369
-
}
370
-
}
371
-
}
+117
tests/main_test.ts
+117
tests/main_test.ts
···
1
+
import { assertEquals } from "@std/assert";
2
+
import { assertMatch } from "@std/assert/match";
3
+
import { createApp } from "../main.ts";
4
+
import { AppContext } from "../context.ts";
5
+
import { Database } from "../data-plane/db/index.ts";
6
+
import { createAuthVerifier } from "../auth-verifier.ts";
7
+
import { getLogger } from "@logtape/logtape";
8
+
import { DataPlane } from "../data-plane/index.ts";
9
+
import { Hydrator } from "../hydration/index.ts";
10
+
import { Views } from "../views/index.ts";
11
+
import { IdResolver } from "@atp/identity";
12
+
import { ServerConfig } from "../config.ts";
13
+
14
+
const cfg = new ServerConfig({
15
+
relayUrl: "http://localhost:8080",
16
+
serverDid: "did:web:localhost",
17
+
modServiceDid: "did:web:test",
18
+
adminPasswords: ["test"],
19
+
privateKey:
20
+
"5676df35fd3a185a1771a43536635ad90057e0c0d1fd91436344bb50ce23a460",
21
+
publicUrl: "http://localhost:4000",
22
+
alternateAudienceDids: [],
23
+
bigThreadUris: new Set(["did:web:test"]),
24
+
maxThreadParents: 10,
25
+
});
26
+
27
+
// Create a mock context for testing without database
28
+
function createMockContext(): AppContext {
29
+
const appLogger = getLogger(["appview"]);
30
+
const idResolver = new IdResolver();
31
+
const serviceDid = "did:web:test";
32
+
33
+
// Create mock database that doesn't actually connect
34
+
const mockDb = {
35
+
connect: () => Promise.resolve(),
36
+
disconnect: () => Promise.resolve(),
37
+
models: {},
38
+
getCursorState: () => Promise.resolve(null),
39
+
saveCursorState: () => Promise.resolve(),
40
+
} as unknown as Database;
41
+
42
+
const dataplane = new DataPlane(mockDb, idResolver);
43
+
const hydrator = new Hydrator(dataplane);
44
+
const views = new Views(cfg);
45
+
const authVerifier = createAuthVerifier(dataplane, {
46
+
ownDid: serviceDid,
47
+
alternateAudienceDids: [],
48
+
modServiceDid: "did:web:test",
49
+
adminPasses: ["test"],
50
+
});
51
+
52
+
return {
53
+
db: mockDb,
54
+
dataplane,
55
+
hydrator,
56
+
views,
57
+
logger: appLogger,
58
+
idResolver,
59
+
cfg,
60
+
authVerifier,
61
+
};
62
+
}
63
+
64
+
Deno.test("Basic App Creation", async () => {
65
+
console.log("Testing basic app creation...");
66
+
67
+
const ctx = createMockContext();
68
+
const app = createApp(ctx);
69
+
70
+
console.log("App created successfully");
71
+
72
+
const res = await app.request("/", {
73
+
headers: {
74
+
"Content-Type": "application/json",
75
+
},
76
+
});
77
+
78
+
assertEquals(res.status, 200);
79
+
console.log("Basic app test passed");
80
+
});
81
+
82
+
Deno.test("Well Known Endpoint", async () => {
83
+
console.log("Testing well-known endpoint...");
84
+
85
+
const ctx = createMockContext();
86
+
const app = createApp(ctx);
87
+
88
+
const res = await app.request("/.well-known/did.json", {
89
+
headers: {
90
+
"Content-Type": "application/json",
91
+
},
92
+
});
93
+
94
+
assertMatch(
95
+
await res.text(),
96
+
new RegExp(
97
+
[
98
+
"^\\{",
99
+
'"@context":\\["https://www\\.w3\\.org/ns/did/v1","https://w3id\\.org/security/multikey/v1"\\],',
100
+
'"id":"(did:web:[^"]+)",',
101
+
'"verificationMethod":\\[\\{',
102
+
'"id":"\\1#atproto",',
103
+
'"type":"Multikey",',
104
+
'"controller":"\\1",',
105
+
'"publicKeyMultibase":"[a-zA-Z0-9]+"',
106
+
"\\}\\],",
107
+
'"service":\\[\\{',
108
+
'"id":"#sprk_appview",',
109
+
'"type":"SprkAppView",',
110
+
'"serviceEndpoint":"https?://[^"]+"',
111
+
"\\}\\]",
112
+
"\\}$",
113
+
].join(""),
114
+
),
115
+
);
116
+
console.log("Well-known endpoint test passed");
117
+
});
+1
-1
utils/audio-transformer.ts
+1
-1
utils/audio-transformer.ts
···
1
1
import type * as SoSprkSoundDefs from "../lex/types/so/sprk/sound/defs.ts";
2
2
import { AudioDocument } from "../data-plane/db/models.ts";
3
-
import { AppContext } from "../main.ts";
3
+
import { AppContext } from "../context.ts";
4
4
import { createProfileViewBasic } from "./profile-helper.ts";
5
5
import type { Label } from "../lex/types/com/atproto/label/defs.ts";
6
6
+13
-5
utils/embed-transformer.ts
+13
-5
utils/embed-transformer.ts
···
4
4
PostEmbed,
5
5
VideoMappingDocument,
6
6
} from "../data-plane/db/models.ts";
7
-
import { env } from "./env.ts";
7
+
import { ServerConfig } from "../config.ts";
8
8
9
9
interface ImageTransformOptions {
10
10
/** If true, only return the first image (useful for stories) */
···
44
44
export function transformVideoEmbed(
45
45
embed: PostEmbed,
46
46
authorDid: string,
47
+
cfg: ServerConfig,
47
48
videoMapping?: VideoMappingDocument | null,
48
49
isStory = false,
49
50
) {
···
55
56
let thumbnail: string;
56
57
57
58
if (videoMapping) {
58
-
playlist = `${env.HLS_CDN_URL}/${videoMapping.bunnyGuid}/playlist.m3u8`;
59
-
thumbnail = `${env.HLS_CDN_URL}/${videoMapping.bunnyGuid}/thumbnail.jpg`;
59
+
playlist = `${cfg.hlsCdn}/${videoMapping.bunnyGuid}/playlist.m3u8`;
60
+
thumbnail = `${cfg.hlsCdn}/${videoMapping.bunnyGuid}/thumbnail.jpg`;
60
61
} else if (isStory) {
61
62
playlist =
62
63
`https://media.sprk.so/video/${authorDid}/${embed.video.ref.$link}`;
···
64
65
`https://thumb.sprk.so/${authorDid}/${embed.video.ref.$link}/thumbnail`;
65
66
} else {
66
67
playlist =
67
-
`${env.VIDEO_CDN_URL}/watch/${authorDid}/${embed.video.ref.$link}/playlist.m3u8`;
68
+
`${cfg.videoCdn}/watch/${authorDid}/${embed.video.ref.$link}/playlist.m3u8`;
68
69
thumbnail =
69
70
`https://thumb.sprk.so/${authorDid}/${embed.video.ref.$link}/thumbnail`;
70
71
}
···
81
82
export function transformEmbed(
82
83
embed: PostEmbed | null,
83
84
authorDid: string,
85
+
cfg: ServerConfig,
84
86
videoMapping?: VideoMappingDocument | null,
85
87
options: ImageTransformOptions = {},
86
88
isStory = false,
···
94
96
}
95
97
96
98
if (embed.$type === "so.sprk.embed.video") {
97
-
return transformVideoEmbed(embed, authorDid, videoMapping, isStory);
99
+
return transformVideoEmbed(
100
+
embed,
101
+
authorDid,
102
+
cfg,
103
+
videoMapping,
104
+
isStory,
105
+
);
98
106
}
99
107
100
108
return undefined;
-32
utils/env.ts
-32
utils/env.ts
···
1
-
import * as dotenv from "dotenv";
2
-
import { envInt, envStr } from "@atp/common";
3
-
4
-
dotenv.config({ quiet: true });
5
-
6
-
export const env = {
7
-
NODE_ENV: envStr("NODE_ENV") ?? "development",
8
-
HOST: envStr("HOST") ?? "0.0.0.0",
9
-
PORT: envInt("PORT") ?? 3000,
10
-
PUBLIC_URL: envStr("PUBLIC_URL") ?? "",
11
-
APPVIEW_K256_PRIVATE_KEY_HEX: envStr("APPVIEW_K256_PRIVATE_KEY_HEX") ?? "",
12
-
SERVICE_DID: envStr("SERVICE_DID") ?? "did:web:localhost",
13
-
MOD_SERVICE_DID: envStr("MOD_SERVICE_DID") ?? "did:web:localhost",
14
-
ADMIN_PASSWORD: envStr("ADMIN_PASSWORD") ?? "admin-token",
15
-
HLS_CDN_URL: envStr("HLS_CDN_URL") ?? "https://vz-fb7436e9-c53.b-cdn.net",
16
-
VIDEO_CDN_URL: envStr("VIDEO_CDN_URL") ?? "https://hls.sprk.so",
17
-
MEDIA_CDN_URL: envStr("MEDIA_CDN_URL") ?? "https://media.sprk.so",
18
-
THUMB_CDN_URL: envStr("THUMB_CDN_URL") ?? "https://thumb.sprk.so",
19
-
20
-
RELAY_URL: envStr("RELAY_URL") ?? "wss://relay1.us-east.bsky.network",
21
-
22
-
DB_URI: envStr("DB_URI"),
23
-
DB_NAME: envStr("DB_NAME") ?? "dev",
24
-
DB_HOST: envStr("DB_HOST") ?? "localhost",
25
-
DB_PORT: envInt("DB_PORT") ?? 27017,
26
-
DB_USER: envStr("DB_USER") ?? "mongo",
27
-
DB_PASSWORD: envStr("DB_PASSWORD") ?? "mongo",
28
-
29
-
RUNNER_CONCURRENCY: envInt("RUNNER_CONCURRENCY") ?? 64,
30
-
BACKGROUND_CONCURRENCY: envInt("BACKGROUND_CONCURRENCY") ?? 16,
31
-
MONGO_MAX_POOL_SIZE: envInt("MONGO_MAX_POOL_SIZE") ?? 50,
32
-
};
-60
utils/id-resolver.ts
-60
utils/id-resolver.ts
···
1
-
import { AtprotoData, IdResolver, MemoryCache } from "@atp/identity";
2
-
3
-
const HOUR = 60e3 * 60;
4
-
const DAY = HOUR * 24;
5
-
6
-
export function createIdResolver() {
7
-
return new IdResolver({
8
-
didCache: new MemoryCache(HOUR, DAY),
9
-
});
10
-
}
11
-
12
-
export interface BidirectionalResolver {
13
-
baseResolver: IdResolver;
14
-
resolveDidToHandle(did: string): Promise<string>;
15
-
resolveDidToDidDoc(did: string): Promise<AtprotoData>;
16
-
resolveHandleToDidDoc(handle: string): Promise<AtprotoData>;
17
-
resolveDidsToHandles(dids: string[]): Promise<Record<string, string>>;
18
-
}
19
-
20
-
export function createBidirectionalResolver(resolver: IdResolver) {
21
-
return {
22
-
baseResolver: resolver,
23
-
24
-
async resolveDidToHandle(did: string): Promise<string> {
25
-
const didDoc = await resolver.did.resolveAtprotoData(did);
26
-
const resolvedDid = await resolver.handle.resolve(didDoc.handle);
27
-
if (resolvedDid === did) {
28
-
return didDoc.handle;
29
-
}
30
-
return "unknown.invalid";
31
-
},
32
-
33
-
async resolveDidToDidDoc(did: string): Promise<AtprotoData> {
34
-
const didDoc = await resolver.did.resolveAtprotoData(did);
35
-
return didDoc;
36
-
},
37
-
38
-
async resolveHandleToDidDoc(handle: string): Promise<AtprotoData> {
39
-
const did = await resolver.handle.resolve(handle);
40
-
if (!did) {
41
-
throw new Error("Handle not found");
42
-
}
43
-
const didDoc = await resolver.did.resolveAtprotoData(did);
44
-
return didDoc;
45
-
},
46
-
47
-
async resolveDidsToHandles(
48
-
dids: string[],
49
-
): Promise<Record<string, string>> {
50
-
const didHandleMap: Record<string, string> = {};
51
-
const resolves = await Promise.all(
52
-
dids.map((did) => this.resolveDidToHandle(did).catch((_) => did)),
53
-
);
54
-
for (let i = 0; i < dids.length; i++) {
55
-
didHandleMap[dids[i]] = resolves[i];
56
-
}
57
-
return didHandleMap;
58
-
},
59
-
};
60
-
}
+27
utils/logger.ts
+27
utils/logger.ts
···
1
+
import { configure, getConsoleSink } from "@logtape/logtape";
2
+
import { getPrettyFormatter } from "@logtape/pretty";
3
+
4
+
export async function configureLogger() {
5
+
await configure({
6
+
sinks: {
7
+
console: getConsoleSink({
8
+
formatter: getPrettyFormatter({
9
+
properties: true,
10
+
categoryStyle: "underline",
11
+
messageColor: "rgb(255, 255, 255)",
12
+
categoryColor: "rgb(255, 255, 255)",
13
+
messageStyle: "reset",
14
+
}),
15
+
}),
16
+
},
17
+
loggers: [
18
+
{ category: "appview", lowestLevel: "info", sinks: ["console"] },
19
+
{ category: "ingester", lowestLevel: "info", sinks: ["console"] },
20
+
{
21
+
category: ["logtape", "meta"],
22
+
lowestLevel: "error",
23
+
sinks: ["console"],
24
+
},
25
+
],
26
+
});
27
+
}
+2
-1
utils/post-transformer.ts
+2
-1
utils/post-transformer.ts
···
2
2
import type { Label } from "../lex/types/com/atproto/label/defs.ts";
3
3
import type * as SoSprkFeedDefs from "../lex/types/so/sprk/feed/defs.ts";
4
4
import type * as SoSprkFeedPost from "../lex/types/so/sprk/feed/post.ts";
5
-
import { AppContext } from "../main.ts";
5
+
import { AppContext } from "../context.ts";
6
6
import { transformAudiosToAudioViews } from "./audio-transformer.ts";
7
7
import { transformEmbed } from "./embed-transformer.ts";
8
8
import { createProfileViewBasic } from "./profile-helper.ts";
···
120
120
const embed = transformEmbed(
121
121
post.embed,
122
122
post.authorDid,
123
+
ctx.cfg,
123
124
videoMapping,
124
125
);
125
126
+34
-56
utils/profile-helper.ts
+34
-56
utils/profile-helper.ts
···
9
9
import type { StoryDocument } from "../data-plane/db/models.ts";
10
10
import type { Label } from "../lex/types/com/atproto/label/defs.ts";
11
11
import { ensureValidDid, isValidHandle } from "@atp/syntax";
12
-
import { AppContext } from "../main.ts";
12
+
import { AppContext } from "../context.ts";
13
13
import { XRPCError } from "@atp/xrpc-server";
14
14
15
-
// Helper function to create ProfileViewBasic with stories
15
+
// Helper function to resolve an actor identifier (handle or DID),
16
+
// fetch profile data, and return a detailed profile view or null if not found
16
17
export async function createProfileViewBasic(
17
18
authorDid: string,
18
19
ctx: AppContext,
···
82
83
actorDid: string,
83
84
viewerDid?: string,
84
85
): Promise<ProfileView> {
85
-
const { db, resolver } = ctx;
86
+
const { db, idResolver } = ctx;
86
87
87
88
const profile = await db.models.Profile.findOne({ authorDid: actorDid });
88
89
const actor = await db.models.Actor.findOne({ did: actorDid });
89
90
90
91
const handle = actor?.handle ??
91
-
(await resolver.resolveDidToHandle(actorDid)) ?? "unknown.invalid";
92
+
(await idResolver.did.resolveAtprotoData(actorDid)).handle ??
93
+
"unknown.invalid";
92
94
93
95
const baseView: ProfileView = {
94
96
$type: "so.sprk.actor.defs#profileView",
···
185
187
const actor = actorMap.get(actorDid);
186
188
187
189
const handle = actor?.handle ??
188
-
(await ctx.resolver.resolveDidToHandle(actorDid)) ?? "unknown.invalid";
190
+
(await ctx.idResolver.did.resolveAtprotoData(actorDid)).handle ??
191
+
"unknown.invalid";
189
192
190
193
const baseView: ProfileView = {
191
194
$type: "so.sprk.actor.defs#profileView",
···
256
259
if (!actorParams || actorParams.length === 0) {
257
260
return [];
258
261
}
259
-
260
-
const now = new Date().toISOString();
261
-
262
262
// Helper function to get a single profile data
263
263
const getProfileData = async (
264
264
actorParam: string,
···
267
267
// Resolve actor identifier to DID
268
268
let actorDidDoc;
269
269
if (isValidHandle(actorParam)) {
270
-
actorDidDoc = await ctx.resolver.resolveHandleToDidDoc(actorParam);
270
+
const did = await ctx.idResolver.handle.resolve(actorParam);
271
+
if (!did) {
272
+
return null; // Invalid handle, skip
273
+
}
274
+
actorDidDoc = await ctx.idResolver.did.resolveAtprotoData(did);
271
275
} else {
272
276
try {
273
277
ensureValidDid(actorParam);
274
-
actorDidDoc = await ctx.resolver.resolveDidToDidDoc(actorParam);
278
+
actorDidDoc = await ctx.idResolver.did.resolveAtprotoData(actorParam);
275
279
} catch (_err) {
276
280
return null; // Invalid actor, skip
277
281
}
···
285
289
ctx.db.models.Profile.findOne({ authorDid: actorDid }),
286
290
]);
287
291
288
-
// If actor doesn't exist, try to index and refetch
289
-
let finalActorDoc = actorDoc;
290
-
let finalProfile = profile;
291
-
292
292
if (!actorDoc) {
293
-
try {
294
-
ctx.logger.info(
295
-
"No actor found, attempting to index",
296
-
{ did: actorDid },
297
-
);
298
-
await ctx.sub.indexingSvc.indexHandle(actorDid, now, true);
299
-
300
-
// Refetch both actor and profile after indexing
301
-
const [refetchedActor, refetchedProfile] = await Promise.all([
302
-
ctx.db.models.Actor.findOne({ did: actorDid }),
303
-
ctx.db.models.Profile.findOne({ authorDid: actorDid }),
304
-
]);
305
-
306
-
finalActorDoc = refetchedActor;
307
-
finalProfile = refetchedProfile;
308
-
} catch (error) {
309
-
ctx.logger.error("Failed to index handle", { error, did: actorDid });
310
-
return null;
311
-
}
312
-
}
313
-
314
-
if (!finalActorDoc) {
315
293
return null; // Actor not found, skip
316
294
}
317
295
318
296
// Handle case where actor exists but profile doesn't
319
-
if (!finalProfile) {
297
+
if (!profile) {
320
298
ctx.logger.info(
321
299
"Actor found but no profile record, creating basic profile view",
322
300
{ did: actorDid },
323
301
);
324
302
325
303
// Get handle
326
-
const handle = finalActorDoc.handle ||
327
-
(await ctx.resolver.resolveDidToHandle(actorDid));
304
+
const handle = actorDoc.handle ||
305
+
((await ctx.idResolver.did.resolveAtprotoData(actorDid)).handle);
328
306
329
307
// Convert to detailed format with minimal data
330
308
return {
···
334
312
}
335
313
336
314
// Get actor's handle and preferences
337
-
const handle = finalActorDoc.handle ||
338
-
(await ctx.resolver.resolveDidToHandle(actorDid));
315
+
const handle = actorDoc.handle ||
316
+
(await ctx.idResolver.did.resolveAtprotoData(actorDid)).handle;
339
317
340
318
// Twenty-four hours ago for recent stories
341
319
const twentyFourHoursAgo = new Date();
···
451
429
452
430
try {
453
431
if (
454
-
finalProfile.avatar && typeof finalProfile.avatar === "object" &&
455
-
finalProfile.avatar.ref && finalProfile.avatar.ref.$link
432
+
profile.avatar && typeof profile.avatar === "object" &&
433
+
profile.avatar.ref && profile.avatar.ref.$link
456
434
) {
457
435
avatar =
458
-
`https://media.sprk.so/avatar/tiny/${actorDid}/${finalProfile.avatar.ref.$link}/webp`;
436
+
`https://media.sprk.so/avatar/tiny/${actorDid}/${profile.avatar.ref.$link}/webp`;
459
437
}
460
438
} catch (error) {
461
439
console.warn(`Failed to construct avatar URL for ${actorDid}:`, error);
···
463
441
464
442
try {
465
443
if (
466
-
finalProfile.banner && typeof finalProfile.banner === "object" &&
467
-
finalProfile.banner.ref && finalProfile.banner.ref.$link
444
+
profile.banner && typeof profile.banner === "object" &&
445
+
profile.banner.ref && profile.banner.ref.$link
468
446
) {
469
447
banner =
470
-
`https://media.sprk.so/img/tiny/${actorDid}/${finalProfile.banner.ref.$link}/webp`;
448
+
`https://media.sprk.so/img/tiny/${actorDid}/${profile.banner.ref.$link}/webp`;
471
449
}
472
450
} catch (error) {
473
451
console.warn(`Failed to construct banner URL for ${actorDid}:`, error);
···
475
453
476
454
// Convert labels to the correct type if it exists
477
455
let labels: Label[] | undefined = undefined;
478
-
if (finalProfile.labels) {
479
-
labels = Array.isArray(finalProfile.labels)
480
-
? (finalProfile.labels as Label[])
456
+
if (profile.labels) {
457
+
labels = Array.isArray(profile.labels)
458
+
? (profile.labels as Label[])
481
459
: undefined;
482
460
}
483
461
484
462
// Convert pinnedPost to the correct type if it exists
485
463
let pinnedPost: ComAtprotoRepoStrongRef.Main | undefined = undefined;
486
-
if (finalProfile.pinnedPost) {
487
-
pinnedPost = finalProfile
464
+
if (profile.pinnedPost) {
465
+
pinnedPost = profile
488
466
.pinnedPost as unknown as ComAtprotoRepoStrongRef.Main;
489
467
}
490
468
···
501
479
const profileView: ProfileViewDetailed = {
502
480
did: actorDid,
503
481
handle: handle,
504
-
displayName: finalProfile.displayName,
505
-
description: finalProfile.description,
482
+
displayName: profile.displayName,
483
+
description: profile.description,
506
484
avatar,
507
485
banner,
508
486
followersCount: typeof followersCount === "number" ? followersCount : 0,
509
487
followsCount: typeof followsCount === "number" ? followsCount : 0,
510
488
postsCount: typeof postsCount === "number" ? postsCount : 0,
511
489
associated: Object.keys(associated).length > 0 ? associated : undefined,
512
-
indexedAt: finalProfile.indexedAt,
513
-
createdAt: finalProfile.createdAt,
490
+
indexedAt: profile.indexedAt,
491
+
createdAt: profile.createdAt,
514
492
viewer: Object.keys(viewer).length > 0 ? viewer : undefined,
515
493
labels,
516
494
pinnedPost,
+21
-7
utils/story-transformer.ts
+21
-7
utils/story-transformer.ts
···
2
2
import { StoryDocument } from "../data-plane/db/models.ts";
3
3
import { transformEmbed } from "./embed-transformer.ts";
4
4
import { createProfileViewBasic } from "./profile-helper.ts";
5
-
import { AppContext } from "../main.ts";
5
+
import { AppContext } from "../context.ts";
6
6
7
7
// Transform DB story to StoryView format
8
8
export async function transformStoryToStoryView(
···
15
15
ctx,
16
16
);
17
17
18
-
const embedView = transformEmbed(story.media, story.authorDid, null, {
19
-
firstImageOnly: true,
20
-
}, true);
18
+
const embedView = transformEmbed(
19
+
story.media,
20
+
story.authorDid,
21
+
ctx.cfg,
22
+
null,
23
+
{
24
+
firstImageOnly: true,
25
+
},
26
+
true,
27
+
);
21
28
22
29
return {
23
30
uri: story.uri,
···
59
66
return stories.map((story) => {
60
67
const authorView = authorsMap.get(story.authorDid)!;
61
68
62
-
const embedView = transformEmbed(story.media, story.authorDid, null, {
63
-
firstImageOnly: true,
64
-
}, true);
69
+
const embedView = transformEmbed(
70
+
story.media,
71
+
story.authorDid,
72
+
ctx.cfg,
73
+
null,
74
+
{
75
+
firstImageOnly: true,
76
+
},
77
+
true,
78
+
);
65
79
66
80
return {
67
81
uri: story.uri,
+20
-9
views/index.ts
+20
-9
views/index.ts
···
32
32
import { INVALID_HANDLE } from "@atp/syntax";
33
33
import { cidFromBlobJson } from "./util.ts";
34
34
import { uriToDid } from "../utils/uris.ts";
35
-
import { env } from "../utils/env.ts";
36
35
import { mapDefined } from "@atp/common";
37
36
import { FeedItem, Repost } from "../hydration/feed.ts";
38
37
import { $Typed } from "../lex/util.ts";
···
40
39
export class Views {
41
40
public indexedAtEpoch?: Date | undefined;
42
41
42
+
private videoCdn?: string;
43
+
private hlsCdn?: string;
44
+
private mediaCdn?: string;
45
+
private thumbCdn?: string;
46
+
43
47
constructor(
44
-
opts?: {
48
+
opts: {
45
49
indexedAtEpoch?: Date | undefined;
50
+
videoCdn?: string;
51
+
hlsCdn?: string;
52
+
mediaCdn?: string;
53
+
thumbCdn?: string;
46
54
},
47
55
) {
48
56
this.indexedAtEpoch = opts?.indexedAtEpoch;
57
+
this.videoCdn = opts?.videoCdn;
58
+
this.hlsCdn = opts?.hlsCdn;
59
+
this.mediaCdn = opts?.mediaCdn;
49
60
}
50
61
51
62
post(
···
293
304
handle: actor.handle ?? INVALID_HANDLE,
294
305
displayName: actor.profile?.displayName,
295
306
avatar: actor.profile?.avatar
296
-
? `${env.MEDIA_CDN_URL}/avatar/medium/${did}/${actor.profile.avatar.ref}/webp`
307
+
? `${this.mediaCdn}/avatar/medium/${did}/${actor.profile.avatar.ref}/webp`
297
308
: undefined,
298
309
viewer: this.profileViewer(did, state),
299
310
createdAt: actor.createdAt?.toISOString(),
···
380
391
embed: ImagesEmbed,
381
392
): ImagesEmbedView & { $type: string } {
382
393
const imgViews = embed.images.map((img) => ({
383
-
thumb: `${env.MEDIA_CDN_URL}/img/medium/${did}/${
394
+
thumb: `${this.mediaCdn}/img/medium/${did}/${
384
395
cidFromBlobJson(img.image)
385
396
}/webp`,
386
-
fullsize: `${env.MEDIA_CDN_URL}/img/full/${did}/${
397
+
fullsize: `${this.mediaCdn}/img/full/${did}/${
387
398
cidFromBlobJson(img.image)
388
399
}/webp`,
389
400
alt: img.alt,
···
406
417
let thumbnail: string;
407
418
408
419
if (videoMapping) {
409
-
playlist = `${env.HLS_CDN_URL}/${videoMapping.bunnyGuid}/playlist.m3u8`;
410
-
thumbnail = `${env.HLS_CDN_URL}/${videoMapping.bunnyGuid}/thumbnail.jpg`;
420
+
playlist = `${this.hlsCdn}/${videoMapping.bunnyGuid}/playlist.m3u8`;
421
+
thumbnail = `${this.hlsCdn}/${videoMapping.bunnyGuid}/thumbnail.jpg`;
411
422
} else {
412
-
playlist = `${env.VIDEO_CDN_URL}/watch/${did}/${cid}/playlist.m3u8`;
413
-
thumbnail = `${env.THUMB_CDN_URL}/${did}/${cid}/thumbnail`;
423
+
playlist = `${this.videoCdn}/watch/${did}/${cid}/playlist.m3u8`;
424
+
thumbnail = `${this.thumbCdn}/${did}/${cid}/thumbnail`;
414
425
}
415
426
416
427
return {