[READ ONLY MIRROR] Spark Social AppView Server
github.com/sprksocial/server
atproto
deno
hono
lexicon
moderation!
+975
-60
+4
-7
lexicons/so/sprk/feed/defs.json
+4
-7
lexicons/so/sprk/feed/defs.json
···
15
15
"record": { "type": "unknown" },
16
16
"embed": {
17
17
"type": "union",
18
-
"refs": [
19
-
"so.sprk.embed.images#view",
20
-
"so.sprk.embed.video#view"
21
-
]
18
+
"refs": ["so.sprk.embed.images#view", "so.sprk.embed.video#view"]
22
19
},
23
-
"sound": { "type": "ref", "ref": "#soundView"},
20
+
"sound": { "type": "ref", "ref": "#soundView" },
24
21
"replyCount": { "type": "integer" },
25
22
"repostCount": { "type": "integer" },
26
23
"likeCount": { "type": "integer" },
27
-
"lookCount": {"type": "integer"},
24
+
"lookCount": { "type": "integer" },
28
25
"indexedAt": { "type": "string", "format": "datetime" },
29
26
"viewer": { "type": "ref", "ref": "#viewerState" },
30
27
"labels": {
···
45
42
"ref": "so.sprk.actor.defs#profileViewBasic"
46
43
},
47
44
"record": { "type": "unknown" },
48
-
"useCount": { "type": "integer"},
45
+
"useCount": { "type": "integer" },
49
46
"likeCount": { "type": "integer" },
50
47
"indexedAt": { "type": "string", "format": "datetime" },
51
48
"labels": {
+50
services/appview/src/auth/middleware.ts
+50
services/appview/src/auth/middleware.ts
···
2
2
import { HTTPException } from 'hono/http-exception'
3
3
import { verifyJwt } from '@atproto/xrpc-server'
4
4
import { DidResolver } from '@atproto/identity'
5
+
import { env } from '../env.js'
6
+
import { TakedownService } from '../services/takedown.js'
5
7
6
8
// Extend the Context type to include auth information
7
9
declare module 'hono' {
···
10
12
accessJwt: string
11
13
serviceDid: string
12
14
didResolver: DidResolver
15
+
isAdmin: boolean
16
+
takedownService: TakedownService
13
17
}
14
18
}
15
19
···
70
74
}
71
75
72
76
await next()
77
+
}
78
+
79
+
// Admin authentication middleware
80
+
export const adminAuthMiddleware = async (c: Context, next: Next) => {
81
+
const authHeader = c.req.header('Authorization')
82
+
const adminPassword = c.req.header('X-Admin-Password')
83
+
84
+
if (!authHeader || !authHeader.startsWith('Bearer ')) {
85
+
throw new HTTPException(401, {
86
+
message: 'Unauthorized: Invalid or missing Authorization header',
87
+
})
88
+
}
89
+
90
+
const jwt = authHeader.replace('Bearer ', '').trim()
91
+
92
+
try {
93
+
// The service DID and resolver should be passed from app context
94
+
const serviceDid = c.get('serviceDid')
95
+
const didResolver = c.get('didResolver') as DidResolver
96
+
97
+
const parsed = await verifyJwt(jwt, serviceDid, null, async (did: string) => {
98
+
return didResolver.resolveAtprotoKey(did)
99
+
})
100
+
101
+
// Set auth information in the context for route handlers to access
102
+
c.set('did', parsed.iss)
103
+
c.set('accessJwt', jwt)
104
+
105
+
// Check if admin password is valid
106
+
if (!adminPassword || !env.ADMIN_PASSWORDS.includes(adminPassword)) {
107
+
throw new HTTPException(403, {
108
+
message: 'Forbidden: Invalid admin credentials',
109
+
})
110
+
}
111
+
112
+
c.set('isAdmin', true)
113
+
114
+
await next()
115
+
} catch (err) {
116
+
if (err instanceof HTTPException) {
117
+
throw err
118
+
}
119
+
throw new HTTPException(401, {
120
+
message: 'Unauthorized: Invalid JWT token',
121
+
})
122
+
}
73
123
}
+61
services/appview/src/db.ts
+61
services/appview/src/db.ts
···
342
342
generatorSchema.index({ authorDid: 1, createdAt: -1 })
343
343
generatorSchema.index({ did: 1, createdAt: -1 })
344
344
345
+
export interface TakedownDocument extends Document {
346
+
targetUri: string
347
+
targetCid: string
348
+
reason: string
349
+
takenDownBy: string
350
+
takenDownAt: string
351
+
}
352
+
353
+
export const takedownSchema = new Schema<TakedownDocument>({
354
+
targetUri: { type: String, required: true, unique: true, index: true },
355
+
targetCid: { type: String, required: true },
356
+
reason: { type: String, required: true },
357
+
takenDownBy: { type: String, required: true },
358
+
takenDownAt: { type: String, required: true },
359
+
})
360
+
361
+
// Repository takedown schema
362
+
export interface RepoTakedownDocument extends Document {
363
+
did: string
364
+
reason: string
365
+
takenDownBy: string
366
+
takenDownAt: string
367
+
ref: string | null
368
+
}
369
+
370
+
export const repoTakedownSchema = new Schema<RepoTakedownDocument>({
371
+
did: { type: String, required: true, unique: true, index: true },
372
+
reason: { type: String, required: true },
373
+
takenDownBy: { type: String, required: true },
374
+
takenDownAt: { type: String, required: true },
375
+
ref: { type: String, required: false, default: null },
376
+
})
377
+
378
+
// Blob takedown schema
379
+
export interface BlobTakedownDocument extends Document {
380
+
did: string
381
+
cid: string
382
+
reason: string
383
+
takenDownBy: string
384
+
takenDownAt: string
385
+
ref: string | null
386
+
}
387
+
388
+
export const blobTakedownSchema = new Schema<BlobTakedownDocument>({
389
+
did: { type: String, required: true, index: true },
390
+
cid: { type: String, required: true, index: true },
391
+
reason: { type: String, required: true },
392
+
takenDownBy: { type: String, required: true },
393
+
takenDownAt: { type: String, required: true },
394
+
ref: { type: String, required: false, default: null },
395
+
})
396
+
397
+
// Ensure compound index on did + cid for blob takedowns
398
+
blobTakedownSchema.index({ did: 1, cid: 1 }, { unique: true })
399
+
345
400
export interface DatabaseModels {
346
401
Like: Model<LikeDocument>
347
402
Post: Model<PostDocument>
···
353
408
Music: Model<MusicDocument>
354
409
Look: Model<LookDocument>
355
410
Generator: Model<GeneratorDocument>
411
+
Takedown: Model<TakedownDocument>
412
+
RepoTakedown: Model<RepoTakedownDocument>
413
+
BlobTakedown: Model<BlobTakedownDocument>
356
414
}
357
415
358
416
export class Database {
···
373
431
Music: this.connection.model<MusicDocument>('Music', musicSchema),
374
432
Look: this.connection.model<LookDocument>('Look', lookSchema),
375
433
Generator: this.connection.model<GeneratorDocument>('Generator', generatorSchema),
434
+
Takedown: this.connection.model<TakedownDocument>('Takedown', takedownSchema),
435
+
RepoTakedown: this.connection.model<RepoTakedownDocument>('RepoTakedown', repoTakedownSchema),
436
+
BlobTakedown: this.connection.model<BlobTakedownDocument>('BlobTakedown', blobTakedownSchema),
376
437
}
377
438
}
378
439
+17
-18
services/appview/src/env.ts
+17
-18
services/appview/src/env.ts
···
1
-
import dotenv from 'dotenv'
2
-
import { cleanEnv, host, port, str, testOnly } from 'envalid'
1
+
import * as dotenv from 'dotenv'
2
+
import { envStr, envList, envInt } from '@atproto/common'
3
3
4
4
dotenv.config()
5
5
6
-
export const env = cleanEnv(process.env, {
7
-
NODE_ENV: str({
8
-
devDefault: testOnly('test'),
9
-
choices: ['development', 'production', 'test'],
10
-
}),
11
-
HOST: host({ devDefault: '0.0.0.0' }),
12
-
PORT: port({ devDefault: 3000 }),
13
-
PUBLIC_URL: str({ devDefault: '' }),
14
-
APPVIEW_K256_PRIVATE_KEY_HEX: str({ devDefault: '' }),
15
-
SERVICE_DID: str({ devDefault: 'did:web:localhost' }),
6
+
export const env = {
7
+
NODE_ENV: envStr('NODE_ENV') ?? 'test',
8
+
HOST: envStr('HOST') ?? '0.0.0.0',
9
+
PORT: envInt('PORT') ?? 3000,
10
+
PUBLIC_URL: envStr('PUBLIC_URL') ?? '',
11
+
APPVIEW_K256_PRIVATE_KEY_HEX: envStr('APPVIEW_K256_PRIVATE_KEY_HEX') ?? '',
12
+
SERVICE_DID: envStr('SERVICE_DID') ?? 'did:web:localhost',
13
+
MOD_SERVICE_DID: envStr('MOD_SERVICE_DID') ?? 'did:web:localhost',
14
+
ADMIN_PASSWORDS: envList('ADMIN_PASSWORDS') ?? [],
16
15
17
-
DB_NAME: str({ devDefault: 'dev' }),
18
-
DB_HOST: str({ devDefault: 'localhost' }),
19
-
DB_PORT: port({ devDefault: 27017 }),
20
-
DB_USER: str({ devDefault: 'mongo' }),
21
-
DB_PASSWORD: str({ devDefault: 'mongo' }),
22
-
})
16
+
DB_NAME: envStr('DB_NAME') ?? 'dev',
17
+
DB_HOST: envStr('DB_HOST') ?? 'localhost',
18
+
DB_PORT: envInt('DB_PORT') ?? 27017,
19
+
DB_USER: envStr('DB_USER') ?? 'mongo',
20
+
DB_PASSWORD: envStr('DB_PASSWORD') ?? 'mongo',
21
+
}
+21
services/appview/src/index.ts
+21
services/appview/src/index.ts
···
19
19
import { createGetFollowersRouter } from './routes/graph/getFollowers.js'
20
20
import { createGetFollowsRouter } from './routes/graph/getFollows.js'
21
21
import wellKnownRouter from './well-known.js'
22
+
import { TakedownService } from './services/takedown.js'
23
+
import takedownRoutes from './routes/admin/takedowns.js'
24
+
import { takedownFilterMiddleware } from './middleware/takedown-filter.js'
22
25
23
26
export type AppContext = {
24
27
db: Database
···
26
29
resolver: BidirectionalResolver
27
30
serviceDid: string
28
31
didResolver: DidResolver
32
+
takedownService: TakedownService
29
33
}
30
34
31
35
export class Server {
···
45
49
46
50
// Get service DID from environment
47
51
const serviceDid = env.SERVICE_DID
52
+
53
+
// Create takedown service
54
+
const takedownService = new TakedownService(db)
48
55
49
56
const ctx = {
50
57
db,
···
52
59
resolver,
53
60
serviceDid,
54
61
didResolver: baseIdResolver.did,
62
+
takedownService,
55
63
}
56
64
57
65
const app = new Hono()
···
64
72
// Type-safe way to set context variables
65
73
c.set('serviceDid', serviceDid)
66
74
c.set('didResolver', baseIdResolver.did)
75
+
c.set('takedownService', takedownService)
67
76
await next()
68
77
})
69
78
···
77
86
const getFollowersRouter = createGetFollowersRouter(ctx)
78
87
const getFollowsRouter = createGetFollowsRouter(ctx)
79
88
const getAuthorFeedRouter = createGetAuthorFeedRouter(ctx)
89
+
90
+
// Apply takedown filter middleware to content routes
91
+
app.use('/xrpc/app.bsky.feed.*', takedownFilterMiddleware)
92
+
app.use('/feed/*', takedownFilterMiddleware)
93
+
app.use('/posts/*', takedownFilterMiddleware)
94
+
80
95
app.route('/', getPostsRouter)
81
96
app.route('/', getPostThreadRouter)
82
97
app.route('/', getProfileRouter)
83
98
app.route('/', getFollowersRouter)
84
99
app.route('/', getFollowsRouter)
85
100
app.route('/', getAuthorFeedRouter)
101
+
102
+
// Admin routes
103
+
app.route('/admin/takedowns', takedownRoutes)
104
+
105
+
// XRPC routes - make sure the Ozone endpoint is accessible
106
+
app.route('/xrpc', takedownRoutes)
86
107
87
108
app.route('/', wellKnownRouter())
88
109
+202
services/appview/src/middleware/takedown-filter.ts
+202
services/appview/src/middleware/takedown-filter.ts
···
1
+
import { Context, Next } from 'hono'
2
+
import { TakedownService } from '../services/takedown.js'
3
+
4
+
/**
5
+
* Middleware that filters out taken-down content from responses
6
+
* This is meant to be applied to routes that return content
7
+
* that might have been taken down by admins
8
+
*/
9
+
export const takedownFilterMiddleware = async (c: Context, next: Next) => {
10
+
// Call the next middleware/route handler first
11
+
await next()
12
+
13
+
// Skip filtering if not a JSON response
14
+
const contentType = c.res.headers.get('Content-Type')
15
+
if (!contentType || !contentType.includes('application/json')) {
16
+
return
17
+
}
18
+
19
+
try {
20
+
// Get the takedown service from context
21
+
const takedownService = c.get('takedownService') as TakedownService
22
+
23
+
// Get the response body
24
+
const body = await c.res.json()
25
+
26
+
// Process different response formats
27
+
if (body.posts && Array.isArray(body.posts)) {
28
+
// For post feeds
29
+
const filteredPosts = await filterTakenDownItems(body.posts, takedownService, 'uri')
30
+
body.posts = filteredPosts
31
+
} else if (body.feed && Array.isArray(body.feed)) {
32
+
// For general feeds
33
+
const filteredFeed = await filterTakenDownItems(body.feed, takedownService, 'post.uri')
34
+
body.feed = filteredFeed
35
+
} else if (body.thread && body.thread.post) {
36
+
// For thread views
37
+
const isThreadTakenDown = await takedownService.isTakenDown(body.thread.post.uri)
38
+
if (isThreadTakenDown) {
39
+
// If the main post is taken down, return empty thread
40
+
body.thread = null
41
+
} else if (body.thread.replies) {
42
+
// Filter replies if they exist
43
+
body.thread.replies = await filterReplies(body.thread.replies, takedownService)
44
+
}
45
+
}
46
+
47
+
// If there are user profiles in the response, filter out taken down repositories
48
+
if (body.profiles && Array.isArray(body.profiles)) {
49
+
const filteredProfiles = await filterTakenDownRepos(body.profiles, takedownService)
50
+
body.profiles = filteredProfiles
51
+
} else if (body.profile) {
52
+
// For single profile view
53
+
if (body.profile.did) {
54
+
const isRepoTakenDown = await takedownService.isRepoTakenDown(body.profile.did)
55
+
if (isRepoTakenDown) {
56
+
body.profile = null
57
+
}
58
+
}
59
+
}
60
+
61
+
// Set the filtered response
62
+
c.res = new Response(JSON.stringify(body), {
63
+
status: c.res.status,
64
+
headers: c.res.headers,
65
+
})
66
+
} catch (error) {
67
+
// In case of error, just continue with the original response
68
+
console.error('Error in takedown filter middleware:', error)
69
+
}
70
+
}
71
+
72
+
// Helper function to filter out taken down items
73
+
async function filterTakenDownItems(
74
+
items: any[],
75
+
takedownService: TakedownService,
76
+
uriPath: string
77
+
): Promise<any[]> {
78
+
if (!items || !Array.isArray(items)) return items
79
+
80
+
const filteredItems: any[] = []
81
+
82
+
for (const item of items) {
83
+
// Get the URI based on the specified path (handles nested objects)
84
+
const uri = uriPath.split('.').reduce((obj, key) => obj && obj[key], item)
85
+
86
+
if (uri) {
87
+
const isTakenDown = await takedownService.isTakenDown(uri)
88
+
89
+
// Check if author's repo is taken down
90
+
let isAuthorTakenDown = false
91
+
if (item.author?.did || (item.post?.author?.did)) {
92
+
const authorDid = item.author?.did || item.post?.author?.did
93
+
isAuthorTakenDown = await takedownService.isRepoTakenDown(authorDid)
94
+
}
95
+
96
+
// Keep the item only if neither the content nor the author is taken down
97
+
if (!isTakenDown && !isAuthorTakenDown) {
98
+
// Filter out taken down images if the item has embeds
99
+
if (item.embed?.images && Array.isArray(item.embed.images)) {
100
+
item.embed.images = await filterTakenDownBlobs(item.embed.images, takedownService)
101
+
}
102
+
103
+
filteredItems.push(item)
104
+
}
105
+
} else {
106
+
// If URI is not found, keep the item
107
+
filteredItems.push(item)
108
+
}
109
+
}
110
+
111
+
return filteredItems
112
+
}
113
+
114
+
// Helper function to filter out taken down repositories
115
+
async function filterTakenDownRepos(
116
+
profiles: any[],
117
+
takedownService: TakedownService
118
+
): Promise<any[]> {
119
+
if (!profiles || !Array.isArray(profiles)) return profiles
120
+
121
+
const filteredProfiles: any[] = []
122
+
123
+
for (const profile of profiles) {
124
+
if (profile.did) {
125
+
const isRepoTakenDown = await takedownService.isRepoTakenDown(profile.did)
126
+
if (!isRepoTakenDown) {
127
+
filteredProfiles.push(profile)
128
+
}
129
+
} else {
130
+
// If no DID, keep the profile
131
+
filteredProfiles.push(profile)
132
+
}
133
+
}
134
+
135
+
return filteredProfiles
136
+
}
137
+
138
+
// Helper function to filter out taken down blobs/images
139
+
async function filterTakenDownBlobs(
140
+
images: any[],
141
+
takedownService: TakedownService
142
+
): Promise<any[]> {
143
+
if (!images || !Array.isArray(images)) return images
144
+
145
+
const filteredImages: any[] = []
146
+
147
+
for (const image of images) {
148
+
// Check if the image is taken down based on blob CID
149
+
if (image.cid && image.did) {
150
+
const isBlobTakenDown = await takedownService.isBlobTakenDown(image.did, image.cid)
151
+
if (!isBlobTakenDown) {
152
+
filteredImages.push(image)
153
+
}
154
+
} else {
155
+
// If no CID or DID, keep the image
156
+
filteredImages.push(image)
157
+
}
158
+
}
159
+
160
+
return filteredImages
161
+
}
162
+
163
+
// Helper function to recursively filter replies in a thread
164
+
async function filterReplies(
165
+
replies: any[],
166
+
takedownService: TakedownService
167
+
): Promise<any[]> {
168
+
if (!replies || !Array.isArray(replies)) return replies
169
+
170
+
const filteredReplies: any[] = []
171
+
172
+
for (const reply of replies) {
173
+
if (reply.post && reply.post.uri) {
174
+
const isTakenDown = await takedownService.isTakenDown(reply.post.uri)
175
+
176
+
// Check if author's repo is taken down
177
+
let isAuthorTakenDown = false
178
+
if (reply.post.author?.did) {
179
+
isAuthorTakenDown = await takedownService.isRepoTakenDown(reply.post.author.did)
180
+
}
181
+
182
+
if (!isTakenDown && !isAuthorTakenDown) {
183
+
// If this reply has nested replies, filter those too
184
+
if (reply.replies && Array.isArray(reply.replies)) {
185
+
reply.replies = await filterReplies(reply.replies, takedownService)
186
+
}
187
+
188
+
// Filter out taken down images in the post
189
+
if (reply.post.embed?.images && Array.isArray(reply.post.embed.images)) {
190
+
reply.post.embed.images = await filterTakenDownBlobs(reply.post.embed.images, takedownService)
191
+
}
192
+
193
+
filteredReplies.push(reply)
194
+
}
195
+
} else {
196
+
// If no post or URI, keep the reply
197
+
filteredReplies.push(reply)
198
+
}
199
+
}
200
+
201
+
return filteredReplies
202
+
}
+328
services/appview/src/routes/admin/takedowns.ts
+328
services/appview/src/routes/admin/takedowns.ts
···
1
+
import { Hono } from 'hono'
2
+
import { zValidator } from '@hono/zod-validator'
3
+
import { z } from 'zod'
4
+
import { HTTPException } from 'hono/http-exception'
5
+
import { TakedownService } from '../../services/takedown.js'
6
+
import { adminAuthMiddleware } from '../../auth/middleware.js'
7
+
8
+
const takedownRoutes = new Hono()
9
+
10
+
// Apply admin auth middleware to all routes
11
+
takedownRoutes.use('*', adminAuthMiddleware)
12
+
13
+
// Create a takedown
14
+
takedownRoutes.post('/', zValidator('json', z.object({
15
+
targetUri: z.string(),
16
+
targetCid: z.string(),
17
+
reason: z.string(),
18
+
})), async (c) => {
19
+
const { targetUri, targetCid, reason } = c.req.valid('json')
20
+
const adminDid = c.get('did')
21
+
22
+
const takedownService = c.get('takedownService') as TakedownService
23
+
24
+
try {
25
+
await takedownService.takedownContent({
26
+
targetUri,
27
+
targetCid,
28
+
reason,
29
+
adminDid,
30
+
})
31
+
32
+
return c.json({ success: true }, 201)
33
+
} catch (error) {
34
+
throw new HTTPException(500, { message: 'Failed to create takedown' })
35
+
}
36
+
})
37
+
38
+
// Remove a takedown (record/post)
39
+
takedownRoutes.delete('/:uri', async (c) => {
40
+
const uri = c.req.param('uri')
41
+
const takedownService = c.get('takedownService') as TakedownService
42
+
43
+
try {
44
+
const removed = await takedownService.removeTakedown(uri)
45
+
46
+
if (!removed) {
47
+
return c.json({ success: false, message: 'Takedown not found' }, 404)
48
+
}
49
+
50
+
return c.json({ success: true })
51
+
} catch (error) {
52
+
throw new HTTPException(500, { message: 'Failed to remove takedown' })
53
+
}
54
+
})
55
+
56
+
// Create a repo takedown
57
+
takedownRoutes.post('/repo', zValidator('json', z.object({
58
+
did: z.string(),
59
+
reason: z.string(),
60
+
ref: z.string().optional(),
61
+
})), async (c) => {
62
+
const { did, reason, ref } = c.req.valid('json')
63
+
const adminDid = c.get('did')
64
+
65
+
const takedownService = c.get('takedownService') as TakedownService
66
+
67
+
try {
68
+
await takedownService.takedownRepo({
69
+
did,
70
+
reason,
71
+
adminDid,
72
+
ref,
73
+
})
74
+
75
+
return c.json({ success: true }, 201)
76
+
} catch (error) {
77
+
throw new HTTPException(500, { message: 'Failed to create repo takedown' })
78
+
}
79
+
})
80
+
81
+
// Remove a repo takedown
82
+
takedownRoutes.delete('/repo/:did', async (c) => {
83
+
const did = c.req.param('did')
84
+
const takedownService = c.get('takedownService') as TakedownService
85
+
86
+
try {
87
+
const removed = await takedownService.removeRepoTakedown(did)
88
+
89
+
if (!removed) {
90
+
return c.json({ success: false, message: 'Repo takedown not found' }, 404)
91
+
}
92
+
93
+
return c.json({ success: true })
94
+
} catch (error) {
95
+
throw new HTTPException(500, { message: 'Failed to remove repo takedown' })
96
+
}
97
+
})
98
+
99
+
// Create a blob takedown
100
+
takedownRoutes.post('/blob', zValidator('json', z.object({
101
+
did: z.string(),
102
+
cid: z.string(),
103
+
reason: z.string(),
104
+
ref: z.string().optional(),
105
+
})), async (c) => {
106
+
const { did, cid, reason, ref } = c.req.valid('json')
107
+
const adminDid = c.get('did')
108
+
109
+
const takedownService = c.get('takedownService') as TakedownService
110
+
111
+
try {
112
+
await takedownService.takedownBlob({
113
+
did,
114
+
cid,
115
+
reason,
116
+
adminDid,
117
+
ref,
118
+
})
119
+
120
+
return c.json({ success: true }, 201)
121
+
} catch (error) {
122
+
throw new HTTPException(500, { message: 'Failed to create blob takedown' })
123
+
}
124
+
})
125
+
126
+
// Remove a blob takedown
127
+
takedownRoutes.delete('/blob/:did/:cid', async (c) => {
128
+
const did = c.req.param('did')
129
+
const cid = c.req.param('cid')
130
+
const takedownService = c.get('takedownService') as TakedownService
131
+
132
+
try {
133
+
const removed = await takedownService.removeBlobTakedown(did, cid)
134
+
135
+
if (!removed) {
136
+
return c.json({ success: false, message: 'Blob takedown not found' }, 404)
137
+
}
138
+
139
+
return c.json({ success: true })
140
+
} catch (error) {
141
+
throw new HTTPException(500, { message: 'Failed to remove blob takedown' })
142
+
}
143
+
})
144
+
145
+
// List takedowns
146
+
takedownRoutes.get('/', zValidator('query', z.object({
147
+
limit: z.string().optional().transform(val => val ? parseInt(val) : 50),
148
+
cursor: z.string().optional(),
149
+
})), async (c) => {
150
+
const { limit, cursor } = c.req.valid('query')
151
+
const takedownService = c.get('takedownService') as TakedownService
152
+
153
+
try {
154
+
const result = await takedownService.listTakedowns(limit, cursor)
155
+
return c.json(result)
156
+
} catch (error) {
157
+
throw new HTTPException(500, { message: 'Failed to list takedowns' })
158
+
}
159
+
})
160
+
161
+
// List repo takedowns
162
+
takedownRoutes.get('/repo', zValidator('query', z.object({
163
+
limit: z.string().optional().transform(val => val ? parseInt(val) : 50),
164
+
cursor: z.string().optional(),
165
+
})), async (c) => {
166
+
const { limit, cursor } = c.req.valid('query')
167
+
const takedownService = c.get('takedownService') as TakedownService
168
+
169
+
try {
170
+
const result = await takedownService.listRepoTakedowns(limit, cursor)
171
+
return c.json(result)
172
+
} catch (error) {
173
+
throw new HTTPException(500, { message: 'Failed to list repo takedowns' })
174
+
}
175
+
})
176
+
177
+
// List blob takedowns
178
+
takedownRoutes.get('/blob', zValidator('query', z.object({
179
+
limit: z.string().optional().transform(val => val ? parseInt(val) : 50),
180
+
cursor: z.string().optional(),
181
+
})), async (c) => {
182
+
const { limit, cursor } = c.req.valid('query')
183
+
const takedownService = c.get('takedownService') as TakedownService
184
+
185
+
try {
186
+
const result = await takedownService.listBlobTakedowns(limit, cursor)
187
+
return c.json(result)
188
+
} catch (error) {
189
+
throw new HTTPException(500, { message: 'Failed to list blob takedowns' })
190
+
}
191
+
})
192
+
193
+
// Check if content is taken down
194
+
takedownRoutes.get('/check/:uri', async (c) => {
195
+
const uri = c.req.param('uri')
196
+
const takedownService = c.get('takedownService') as TakedownService
197
+
198
+
try {
199
+
const isTakenDown = await takedownService.isTakenDown(uri)
200
+
return c.json({ isTakenDown })
201
+
} catch (error) {
202
+
throw new HTTPException(500, { message: 'Failed to check takedown status' })
203
+
}
204
+
})
205
+
206
+
// Check if repo is taken down
207
+
takedownRoutes.get('/check/repo/:did', async (c) => {
208
+
const did = c.req.param('did')
209
+
const takedownService = c.get('takedownService') as TakedownService
210
+
211
+
try {
212
+
const isTakenDown = await takedownService.isRepoTakenDown(did)
213
+
return c.json({ isTakenDown })
214
+
} catch (error) {
215
+
throw new HTTPException(500, { message: 'Failed to check repo takedown status' })
216
+
}
217
+
})
218
+
219
+
// Check if blob is taken down
220
+
takedownRoutes.get('/check/blob/:did/:cid', async (c) => {
221
+
const did = c.req.param('did')
222
+
const cid = c.req.param('cid')
223
+
const takedownService = c.get('takedownService') as TakedownService
224
+
225
+
try {
226
+
const isTakenDown = await takedownService.isBlobTakenDown(did, cid)
227
+
return c.json({ isTakenDown })
228
+
} catch (error) {
229
+
throw new HTTPException(500, { message: 'Failed to check blob takedown status' })
230
+
}
231
+
})
232
+
233
+
// XRPC endpoint for Ozone integration: com.atproto.admin.updateSubjectStatus
234
+
takedownRoutes.post('/xrpc/com.atproto.admin.updateSubjectStatus', zValidator('json', z.object({
235
+
subject: z.object({
236
+
$type: z.string(),
237
+
did: z.string().optional(),
238
+
uri: z.string().optional(),
239
+
cid: z.string().optional(),
240
+
}),
241
+
takedown: z.object({
242
+
applied: z.boolean(),
243
+
ref: z.string().optional(),
244
+
}),
245
+
})), async (c) => {
246
+
const { subject, takedown } = c.req.valid('json')
247
+
const adminDid = c.get('did')
248
+
249
+
const takedownService = c.get('takedownService') as TakedownService
250
+
251
+
try {
252
+
// Handle different subject types
253
+
if (subject.$type === 'com.atproto.admin.defs#repoRef') {
254
+
// Repository (user account) takedown
255
+
if (!subject.did) {
256
+
throw new HTTPException(400, { message: 'DID is required for repo takedowns' })
257
+
}
258
+
259
+
if (takedown.applied) {
260
+
// Apply takedown
261
+
await takedownService.takedownRepo({
262
+
did: subject.did,
263
+
reason: 'Moderation via Ozone',
264
+
adminDid,
265
+
ref: takedown.ref,
266
+
})
267
+
} else {
268
+
// Remove takedown
269
+
await takedownService.removeRepoTakedown(subject.did)
270
+
}
271
+
} else if (subject.$type === 'com.atproto.repo.strongRef') {
272
+
// Record (post) takedown
273
+
if (!subject.uri || !subject.cid) {
274
+
throw new HTTPException(400, { message: 'URI and CID are required for record takedowns' })
275
+
}
276
+
277
+
if (takedown.applied) {
278
+
// Apply takedown
279
+
await takedownService.takedownContent({
280
+
targetUri: subject.uri,
281
+
targetCid: subject.cid,
282
+
reason: 'Moderation via Ozone',
283
+
adminDid,
284
+
})
285
+
} else {
286
+
// Remove takedown
287
+
await takedownService.removeTakedown(subject.uri)
288
+
}
289
+
} else if (subject.$type === 'com.atproto.admin.defs#repoBlobRef') {
290
+
// Blob (image/attachment) takedown
291
+
if (!subject.did || !subject.cid) {
292
+
throw new HTTPException(400, { message: 'DID and CID are required for blob takedowns' })
293
+
}
294
+
295
+
if (takedown.applied) {
296
+
// Apply takedown
297
+
await takedownService.takedownBlob({
298
+
did: subject.did,
299
+
cid: subject.cid,
300
+
reason: 'Moderation via Ozone',
301
+
adminDid,
302
+
ref: takedown.ref,
303
+
})
304
+
} else {
305
+
// Remove takedown
306
+
await takedownService.removeBlobTakedown(subject.did, subject.cid)
307
+
}
308
+
} else {
309
+
throw new HTTPException(400, { message: `Unsupported subject type: ${subject.$type}` })
310
+
}
311
+
312
+
// Return the response format expected by Ozone
313
+
return c.json({
314
+
subject,
315
+
takedown: takedown.applied ? {
316
+
applied: takedown.applied,
317
+
ref: takedown.ref
318
+
} : undefined
319
+
})
320
+
} catch (error) {
321
+
if (error instanceof HTTPException) {
322
+
throw error
323
+
}
324
+
throw new HTTPException(500, { message: 'Failed to update subject status' })
325
+
}
326
+
})
327
+
328
+
export default takedownRoutes
+203
services/appview/src/services/takedown.ts
+203
services/appview/src/services/takedown.ts
···
1
+
import { Database } from '../db.js'
2
+
3
+
export class TakedownService {
4
+
constructor(private db: Database) {}
5
+
6
+
async takedownContent(params: {
7
+
targetUri: string
8
+
targetCid: string
9
+
reason: string
10
+
adminDid: string
11
+
}): Promise<void> {
12
+
const { targetUri, targetCid, reason, adminDid } = params
13
+
14
+
// Create a takedown record
15
+
await this.db.models.Takedown.create({
16
+
targetUri,
17
+
targetCid,
18
+
reason,
19
+
takenDownBy: adminDid,
20
+
takenDownAt: new Date().toISOString(),
21
+
})
22
+
}
23
+
24
+
// Add a method to handle user repo takedowns
25
+
async takedownRepo(params: {
26
+
did: string
27
+
reason: string
28
+
adminDid: string
29
+
ref?: string
30
+
}): Promise<void> {
31
+
const { did, reason, adminDid, ref } = params
32
+
33
+
// Create a repo takedown record
34
+
await this.db.models.RepoTakedown.create({
35
+
did,
36
+
reason,
37
+
takenDownBy: adminDid,
38
+
takenDownAt: new Date().toISOString(),
39
+
ref: ref || null,
40
+
})
41
+
}
42
+
43
+
// Add a method to handle blob takedowns
44
+
async takedownBlob(params: {
45
+
did: string
46
+
cid: string
47
+
reason: string
48
+
adminDid: string
49
+
ref?: string
50
+
}): Promise<void> {
51
+
const { did, cid, reason, adminDid, ref } = params
52
+
53
+
// Create a blob takedown record
54
+
await this.db.models.BlobTakedown.create({
55
+
did,
56
+
cid,
57
+
reason,
58
+
takenDownBy: adminDid,
59
+
takenDownAt: new Date().toISOString(),
60
+
ref: ref || null,
61
+
})
62
+
}
63
+
64
+
async isTakenDown(uri: string): Promise<boolean> {
65
+
const takedown = await this.db.models.Takedown.findOne({ targetUri: uri })
66
+
return !!takedown
67
+
}
68
+
69
+
// Add a method to check if a repo is taken down
70
+
async isRepoTakenDown(did: string): Promise<boolean> {
71
+
const takedown = await this.db.models.RepoTakedown.findOne({ did })
72
+
return !!takedown
73
+
}
74
+
75
+
// Add a method to check if a blob is taken down
76
+
async isBlobTakenDown(did: string, cid: string): Promise<boolean> {
77
+
const takedown = await this.db.models.BlobTakedown.findOne({ did, cid })
78
+
return !!takedown
79
+
}
80
+
81
+
async removeTakedown(targetUri: string): Promise<boolean> {
82
+
const result = await this.db.models.Takedown.deleteOne({ targetUri })
83
+
return result.deletedCount > 0
84
+
}
85
+
86
+
// Add a method to remove repo takedown
87
+
async removeRepoTakedown(did: string): Promise<boolean> {
88
+
const result = await this.db.models.RepoTakedown.deleteOne({ did })
89
+
return result.deletedCount > 0
90
+
}
91
+
92
+
// Add a method to remove blob takedown
93
+
async removeBlobTakedown(did: string, cid: string): Promise<boolean> {
94
+
const result = await this.db.models.BlobTakedown.deleteOne({ did, cid })
95
+
return result.deletedCount > 0
96
+
}
97
+
98
+
async listTakedowns(limit: number = 50, cursor?: string): Promise<{
99
+
takedowns: Array<{
100
+
targetUri: string
101
+
targetCid: string
102
+
reason: string
103
+
takenDownBy: string
104
+
takenDownAt: string
105
+
}>
106
+
cursor?: string
107
+
}> {
108
+
const query = cursor
109
+
? { targetUri: { $lt: cursor } }
110
+
: {}
111
+
112
+
const takedowns = await this.db.models.Takedown
113
+
.find(query)
114
+
.sort({ targetUri: -1 })
115
+
.limit(limit + 1)
116
+
.lean()
117
+
118
+
const items = takedowns.slice(0, limit)
119
+
120
+
return {
121
+
takedowns: items.map(t => ({
122
+
targetUri: t.targetUri,
123
+
targetCid: t.targetCid,
124
+
reason: t.reason,
125
+
takenDownBy: t.takenDownBy,
126
+
takenDownAt: t.takenDownAt
127
+
})),
128
+
cursor: takedowns.length > limit ? takedowns[limit - 1].targetUri : undefined
129
+
}
130
+
}
131
+
132
+
// Add a method to list repo takedowns
133
+
async listRepoTakedowns(limit: number = 50, cursor?: string): Promise<{
134
+
repoTakedowns: Array<{
135
+
did: string
136
+
reason: string
137
+
takenDownBy: string
138
+
takenDownAt: string
139
+
ref: string | null
140
+
}>
141
+
cursor?: string
142
+
}> {
143
+
const query = cursor
144
+
? { did: { $lt: cursor } }
145
+
: {}
146
+
147
+
const takedowns = await this.db.models.RepoTakedown
148
+
.find(query)
149
+
.sort({ did: -1 })
150
+
.limit(limit + 1)
151
+
.lean()
152
+
153
+
const items = takedowns.slice(0, limit)
154
+
155
+
return {
156
+
repoTakedowns: items.map(t => ({
157
+
did: t.did,
158
+
reason: t.reason,
159
+
takenDownBy: t.takenDownBy,
160
+
takenDownAt: t.takenDownAt,
161
+
ref: t.ref
162
+
})),
163
+
cursor: takedowns.length > limit ? takedowns[limit - 1].did : undefined
164
+
}
165
+
}
166
+
167
+
// Add a method to list blob takedowns
168
+
async listBlobTakedowns(limit: number = 50, cursor?: string): Promise<{
169
+
blobTakedowns: Array<{
170
+
did: string
171
+
cid: string
172
+
reason: string
173
+
takenDownBy: string
174
+
takenDownAt: string
175
+
ref: string | null
176
+
}>
177
+
cursor?: string
178
+
}> {
179
+
const query = cursor
180
+
? { did: { $lt: cursor } }
181
+
: {}
182
+
183
+
const takedowns = await this.db.models.BlobTakedown
184
+
.find(query)
185
+
.sort({ did: -1, cid: -1 })
186
+
.limit(limit + 1)
187
+
.lean()
188
+
189
+
const items = takedowns.slice(0, limit)
190
+
191
+
return {
192
+
blobTakedowns: items.map(t => ({
193
+
did: t.did,
194
+
cid: t.cid,
195
+
reason: t.reason,
196
+
takenDownBy: t.takenDownBy,
197
+
takenDownAt: t.takenDownAt,
198
+
ref: t.ref
199
+
})),
200
+
cursor: takedowns.length > limit ? takedowns[limit - 1].did : undefined
201
+
}
202
+
}
203
+
}
+11
-17
services/feed-gen/src/utils/env.ts
+11
-17
services/feed-gen/src/utils/env.ts
···
1
-
import dotenv from 'dotenv'
2
-
import { cleanEnv, port, str, testOnly } from 'envalid'
1
+
import * as dotenv from 'dotenv'
2
+
import { envBool, envInt, envList, envStr } from '@atproto/common'
3
3
4
4
dotenv.config()
5
5
6
-
export const env = cleanEnv(process.env, {
7
-
NODE_ENV: str({
8
-
devDefault: testOnly('test'),
9
-
choices: ['development', 'production', 'test'],
10
-
}),
6
+
export const env = {
7
+
NODE_ENV: envStr('NODE_ENV'),
11
8
12
-
FEEDGEN_DOMAIN: str({
13
-
default: 'example.com',
14
-
desc: 'Domain of the feed generator for did:web'
15
-
}),
9
+
FEEDGEN_DOMAIN: envStr('FEEDGEN_DOMAIN'),
16
10
17
-
DB_NAME: str({ devDefault: 'dev' }),
18
-
DB_HOST: str({ devDefault: 'localhost' }),
19
-
DB_PORT: port({ devDefault: 27017 }),
20
-
DB_USER: str({ devDefault: 'mongo' }),
21
-
DB_PASSWORD: str({ devDefault: 'mongo' }),
22
-
})
11
+
DB_NAME: envStr('DB_NAME'),
12
+
DB_HOST: envStr('DB_HOST'),
13
+
DB_PORT: envInt('DB_PORT'),
14
+
DB_USER: envStr('DB_USER'),
15
+
DB_PASSWORD: envStr('DB_PASSWORD')
16
+
}
+64
-1
services/ingester/bun.lock
+64
-1
services/ingester/bun.lock
···
4
4
"": {
5
5
"name": "ingester",
6
6
"dependencies": {
7
+
"@atproto/common": "^0.4.6",
7
8
"@atproto/identity": "^0.4.6",
8
9
"dotenv": "^16.4.7",
9
10
"envalid": "^8.0.0",
···
22
23
},
23
24
},
24
25
"packages": {
25
-
"@atproto/common-web": ["@atproto/common-web@0.4.0", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, "sha512-ZYL0P9myHybNgwh/hBY0HaBzqiLR1B5/ie5bJpLQAg0whRzNA28t8/nU2vh99tbsWcAF0LOD29M8++LyENJLNQ=="],
26
+
"@atproto/common": ["@atproto/common@0.4.10", "", { "dependencies": { "@atproto/common-web": "^0.4.1", "@ipld/dag-cbor": "^7.0.3", "cbor-x": "^1.5.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "pino": "^8.21.0" } }, "sha512-/Yxnax3XOhf46jYpe8/6O3ORjTNMB4YCaxx3V1f+FKy6meTm3GNrJwo8d1CBs0UiTiheRiNATOV3u0s3C7Ydaw=="],
27
+
28
+
"@atproto/common-web": ["@atproto/common-web@0.4.1", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, "sha512-Ghh+djHYMAUCktLKwr2IuGgtjcwSWGudp+K7+N7KBA9pDDloOXUEY8Agjc5SHSo9B1QIEFkegClU5n+apn2e0w=="],
26
29
27
30
"@atproto/crypto": ["@atproto/crypto@0.4.4", "", { "dependencies": { "@noble/curves": "^1.7.0", "@noble/hashes": "^1.6.1", "uint8arrays": "3.0.0" } }, "sha512-Yq9+crJ7WQl7sxStVpHgie5Z51R05etaK9DLWYG/7bR5T4bhdcIgF6IfklLShtZwLYdVVj+K15s0BqW9a8PSDA=="],
28
31
29
32
"@atproto/identity": ["@atproto/identity@0.4.6", "", { "dependencies": { "@atproto/common-web": "^0.4.0", "@atproto/crypto": "^0.4.4" } }, "sha512-fJq/cIp9MOgHxZfxuyki6mobk0QxRnbts53DstRixlvb5mOoxwttb9Gp6A8u9q49zBsfOmXNTHmP97I9iMHmTQ=="],
30
33
34
+
"@cbor-extract/cbor-extract-darwin-arm64": ["@cbor-extract/cbor-extract-darwin-arm64@2.2.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-P7swiOAdF7aSi0H+tHtHtr6zrpF3aAq/W9FXx5HektRvLTM2O89xCyXF3pk7pLc7QpaY7AoaE8UowVf9QBdh3w=="],
35
+
36
+
"@cbor-extract/cbor-extract-darwin-x64": ["@cbor-extract/cbor-extract-darwin-x64@2.2.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-1liF6fgowph0JxBbYnAS7ZlqNYLf000Qnj4KjqPNW4GViKrEql2MgZnAsExhY9LSy8dnvA4C0qHEBgPrll0z0w=="],
37
+
38
+
"@cbor-extract/cbor-extract-linux-arm": ["@cbor-extract/cbor-extract-linux-arm@2.2.0", "", { "os": "linux", "cpu": "arm" }, "sha512-QeBcBXk964zOytiedMPQNZr7sg0TNavZeuUCD6ON4vEOU/25+pLhNN6EDIKJ9VLTKaZ7K7EaAriyYQ1NQ05s/Q=="],
39
+
40
+
"@cbor-extract/cbor-extract-linux-arm64": ["@cbor-extract/cbor-extract-linux-arm64@2.2.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-rQvhNmDuhjTVXSPFLolmQ47/ydGOFXtbR7+wgkSY0bdOxCFept1hvg59uiLPT2fVDuJFuEy16EImo5tE2x3RsQ=="],
41
+
42
+
"@cbor-extract/cbor-extract-linux-x64": ["@cbor-extract/cbor-extract-linux-x64@2.2.0", "", { "os": "linux", "cpu": "x64" }, "sha512-cWLAWtT3kNLHSvP4RKDzSTX9o0wvQEEAj4SKvhWuOVZxiDAeQazr9A+PSiRILK1VYMLeDml89ohxCnUNQNQNCw=="],
43
+
44
+
"@cbor-extract/cbor-extract-win32-x64": ["@cbor-extract/cbor-extract-win32-x64@2.2.0", "", { "os": "win32", "cpu": "x64" }, "sha512-l2M+Z8DO2vbvADOBNLbbh9y5ST1RY5sqkWOg/58GkUPBYou/cuNZ68SGQ644f1CvZ8kcOxyZtw06+dxWHIoN/w=="],
45
+
46
+
"@ipld/dag-cbor": ["@ipld/dag-cbor@7.0.3", "", { "dependencies": { "cborg": "^1.6.0", "multiformats": "^9.5.4" } }, "sha512-1VVh2huHsuohdXC1bGJNE8WR72slZ9XE2T3wbBBq31dm7ZBatmKLLxrB+XAqafxfRFjv08RZmj/W/ZqaM13AuA=="],
47
+
31
48
"@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.2.0", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-+ywrb0AqkfaYuhHs6LxKWgqbh3I72EpEgESCw37o+9qPx9WTCkgDm2B+eMrwehGtHBWHFU4GXvnSCNiFhhausg=="],
32
49
33
50
"@noble/curves": ["@noble/curves@1.8.1", "", { "dependencies": { "@noble/hashes": "1.7.1" } }, "sha512-warwspo+UYUPep0Q+vtdVB4Ugn8GGQj8iyB3gnRWsztmUHTI3S1nhdiWNsPUGL0vud7JlRRk1XEu7Lq1KGTnMQ=="],
···
44
61
45
62
"@types/ws": ["@types/ws@8.18.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-8svvI3hMyvN0kKCJMvTJP/x6Y/EoQbepff882wL+Sn5QsXb3etnamgrJq4isrBxSJj5L2AuXcI0+bgkoAXGUJw=="],
46
63
64
+
"abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="],
65
+
47
66
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
48
67
68
+
"base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="],
69
+
49
70
"bson": ["bson@6.10.3", "", {}, "sha512-MTxGsqgYTwfshYWTRdmZRC+M7FnG1b4y7RO7p2k3X24Wq0yv1m77Wsj0BzlPzd/IowgESfsruQCUToa7vbOpPQ=="],
71
+
72
+
"buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="],
50
73
51
74
"bun-types": ["bun-types@1.2.5", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-3oO6LVGGRRKI4kHINx5PIdIgnLRb7l/SprhzqXapmoYkFl5m4j6EvALvbDVuuBFaamB46Ap6HCUxIXNLCGy+tg=="],
52
75
76
+
"cbor-extract": ["cbor-extract@2.2.0", "", { "dependencies": { "node-gyp-build-optional-packages": "5.1.1" }, "optionalDependencies": { "@cbor-extract/cbor-extract-darwin-arm64": "2.2.0", "@cbor-extract/cbor-extract-darwin-x64": "2.2.0", "@cbor-extract/cbor-extract-linux-arm": "2.2.0", "@cbor-extract/cbor-extract-linux-arm64": "2.2.0", "@cbor-extract/cbor-extract-linux-x64": "2.2.0", "@cbor-extract/cbor-extract-win32-x64": "2.2.0" }, "bin": { "download-cbor-prebuilds": "bin/download-prebuilds.js" } }, "sha512-Ig1zM66BjLfTXpNgKpvBePq271BPOvu8MR0Jl080yG7Jsl+wAZunfrwiwA+9ruzm/WEdIV5QF/bjDZTqyAIVHA=="],
77
+
78
+
"cbor-x": ["cbor-x@1.6.0", "", { "optionalDependencies": { "cbor-extract": "^2.2.0" } }, "sha512-0kareyRwHSkL6ws5VXHEf8uY1liitysCVJjlmhaLG+IXLqhSaOO+t63coaso7yjwEzWZzLy8fJo06gZDVQM9Qg=="],
79
+
80
+
"cborg": ["cborg@1.10.2", "", { "bin": { "cborg": "cli.js" } }, "sha512-b3tFPA9pUr2zCUiCfRd2+wok2/LBSNUMKOuRRok+WlvvAgEt/PlbgPTsZUcwCOs53IJvLgTp0eotwtosE6njug=="],
81
+
53
82
"colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="],
54
83
55
84
"dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="],
56
85
57
86
"debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],
87
+
88
+
"detect-libc": ["detect-libc@2.0.3", "", {}, "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw=="],
58
89
59
90
"dotenv": ["dotenv@16.4.7", "", {}, "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ=="],
60
91
···
62
93
63
94
"envalid": ["envalid@8.0.0", "", { "dependencies": { "tslib": "2.6.2" } }, "sha512-PGeYJnJB5naN0ME6SH8nFcDj9HVbLpYIfg1p5lAyM9T4cH2lwtu2fLbozC/bq+HUUOIFxhX/LP0/GmlqPHT4tQ=="],
64
95
96
+
"event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="],
97
+
98
+
"events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="],
99
+
65
100
"fast-copy": ["fast-copy@3.0.2", "", {}, "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="],
66
101
67
102
"fast-redact": ["fast-redact@3.5.0", "", {}, "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A=="],
···
71
106
"graphemer": ["graphemer@1.4.0", "", {}, "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="],
72
107
73
108
"help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="],
109
+
110
+
"ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],
111
+
112
+
"iso-datestring-validator": ["iso-datestring-validator@2.2.2", "", {}, "sha512-yLEMkBbLZTlVQqOnQ4FiMujR6T4DEcCb1xizmvXS+OxuhwcbtynoosRzdMA69zZCShCNAbi+gJ71FxZBBXx1SA=="],
74
113
75
114
"joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="],
76
115
···
94
133
95
134
"multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="],
96
135
136
+
"node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.1.1", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-test": "build-test.js", "node-gyp-build-optional-packages-optional": "optional.js" } }, "sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw=="],
137
+
97
138
"on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="],
98
139
99
140
"once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
···
105
146
"pino-pretty": ["pino-pretty@13.0.0", "", { "dependencies": { "colorette": "^2.0.7", "dateformat": "^4.6.3", "fast-copy": "^3.0.2", "fast-safe-stringify": "^2.1.1", "help-me": "^5.0.0", "joycon": "^3.1.1", "minimist": "^1.2.6", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pump": "^3.0.0", "secure-json-parse": "^2.4.0", "sonic-boom": "^4.0.1", "strip-json-comments": "^3.1.1" }, "bin": { "pino-pretty": "bin.js" } }, "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA=="],
106
147
107
148
"pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="],
149
+
150
+
"process": ["process@0.11.10", "", {}, "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A=="],
108
151
109
152
"process-warning": ["process-warning@4.0.1", "", {}, "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q=="],
110
153
···
114
157
115
158
"quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="],
116
159
160
+
"readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="],
161
+
117
162
"real-require": ["real-require@0.2.0", "", {}, "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg=="],
163
+
164
+
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
118
165
119
166
"safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="],
120
167
···
128
175
129
176
"split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="],
130
177
178
+
"string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="],
179
+
131
180
"strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="],
132
181
133
182
"thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="],
···
152
201
153
202
"zod": ["zod@3.24.2", "", {}, "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ=="],
154
203
204
+
"@atproto/common/pino": ["pino@8.21.0", "", { "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^1.2.0", "pino-std-serializers": "^6.0.0", "process-warning": "^3.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^3.7.0", "thread-stream": "^2.6.0" }, "bin": { "pino": "bin.js" } }, "sha512-ip4qdzjkAyDDZklUaZkcRFb2iA118H9SgRh8yzTkSQK8HilsOJF7rSY8HoW5+I0M46AZgX/pxbprf2vvzQCE0Q=="],
205
+
206
+
"@atproto/identity/@atproto/common-web": ["@atproto/common-web@0.4.0", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, "sha512-ZYL0P9myHybNgwh/hBY0HaBzqiLR1B5/ie5bJpLQAg0whRzNA28t8/nU2vh99tbsWcAF0LOD29M8++LyENJLNQ=="],
207
+
155
208
"bun-types/@types/ws": ["@types/ws@8.5.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw=="],
209
+
210
+
"@atproto/common/pino/pino-abstract-transport": ["pino-abstract-transport@1.2.0", "", { "dependencies": { "readable-stream": "^4.0.0", "split2": "^4.0.0" } }, "sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q=="],
211
+
212
+
"@atproto/common/pino/pino-std-serializers": ["pino-std-serializers@6.2.2", "", {}, "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="],
213
+
214
+
"@atproto/common/pino/process-warning": ["process-warning@3.0.0", "", {}, "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="],
215
+
216
+
"@atproto/common/pino/sonic-boom": ["sonic-boom@3.8.1", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg=="],
217
+
218
+
"@atproto/common/pino/thread-stream": ["thread-stream@2.7.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw=="],
156
219
}
157
220
}
+1
services/ingester/package.json
+1
services/ingester/package.json
+13
-17
services/ingester/src/utils/env.ts
+13
-17
services/ingester/src/utils/env.ts
···
1
-
import dotenv from 'dotenv'
2
-
import { cleanEnv, host, port, str, testOnly } from 'envalid'
1
+
import * as dotenv from 'dotenv'
2
+
import { envBool, envInt, envList, envStr } from '@atproto/common'
3
3
4
4
dotenv.config()
5
5
6
-
export const env = cleanEnv(process.env, {
7
-
NODE_ENV: str({
8
-
devDefault: testOnly('test'),
9
-
choices: ['development', 'production', 'test'],
10
-
}),
6
+
export const env = {
7
+
NODE_ENV: envStr('NODE_ENV') ?? 'test',
11
8
12
-
JETSTREAM_URL: str({
13
-
default: 'wss://jetstream2.us-east.bsky.network/subscribe',
14
-
desc: 'Websocket URL for Jetstream connection'
15
-
}),
9
+
JETSTREAM_URL:
10
+
envStr('JETSTREAM_URL') ??
11
+
'wss://jetstream2.us-east.bsky.network/subscribe',
16
12
17
-
DB_NAME: str({ devDefault: 'dev' }),
18
-
DB_HOST: str({ devDefault: 'localhost' }),
19
-
DB_PORT: port({ devDefault: 27017 }),
20
-
DB_USER: str({ devDefault: 'mongo' }),
21
-
DB_PASSWORD: str({ devDefault: 'mongo' }),
22
-
})
13
+
DB_NAME: envStr('DB_NAME') ?? 'dev',
14
+
DB_HOST: envStr('DB_HOST') ?? 'localhost',
15
+
DB_PORT: envInt('DB_PORT') ?? 27017,
16
+
DB_USER: envStr('DB_USER') ?? 'mongo',
17
+
DB_PASSWORD: envStr('DB_PASSWORD') ?? 'mongo',
18
+
}