moderation! (#1) · sprk.so/server@3774ea6

+4 -7

lexicons/so/sprk/feed/defs.json

··· 15 15 "record": { "type": "unknown" }, 16 16 "embed": { 17 17 "type": "union", 18 - "refs": [ 19 - "so.sprk.embed.images#view", 20 - "so.sprk.embed.video#view" 21 - ] 18 + "refs": ["so.sprk.embed.images#view", "so.sprk.embed.video#view"] 22 19 }, 23 - "sound": { "type": "ref", "ref": "#soundView"}, 20 + "sound": { "type": "ref", "ref": "#soundView" }, 24 21 "replyCount": { "type": "integer" }, 25 22 "repostCount": { "type": "integer" }, 26 23 "likeCount": { "type": "integer" }, 27 - "lookCount": {"type": "integer"}, 24 + "lookCount": { "type": "integer" }, 28 25 "indexedAt": { "type": "string", "format": "datetime" }, 29 26 "viewer": { "type": "ref", "ref": "#viewerState" }, 30 27 "labels": { ··· 45 42 "ref": "so.sprk.actor.defs#profileViewBasic" 46 43 }, 47 44 "record": { "type": "unknown" }, 48 - "useCount": { "type": "integer"}, 45 + "useCount": { "type": "integer" }, 49 46 "likeCount": { "type": "integer" }, 50 47 "indexedAt": { "type": "string", "format": "datetime" }, 51 48 "labels": {

+50

services/appview/src/auth/middleware.ts

··· 2 2 import { HTTPException } from 'hono/http-exception' 3 3 import { verifyJwt } from '@atproto/xrpc-server' 4 4 import { DidResolver } from '@atproto/identity' 5 + import { env } from '../env.js' 6 + import { TakedownService } from '../services/takedown.js' 5 7 6 8 // Extend the Context type to include auth information 7 9 declare module 'hono' { ··· 10 12 accessJwt: string 11 13 serviceDid: string 12 14 didResolver: DidResolver 15 + isAdmin: boolean 16 + takedownService: TakedownService 13 17 } 14 18 } 15 19 ··· 70 74 } 71 75 72 76 await next() 77 + } 78 + 79 + // Admin authentication middleware 80 + export const adminAuthMiddleware = async (c: Context, next: Next) => { 81 + const authHeader = c.req.header('Authorization') 82 + const adminPassword = c.req.header('X-Admin-Password') 83 + 84 + if (!authHeader || !authHeader.startsWith('Bearer ')) { 85 + throw new HTTPException(401, { 86 + message: 'Unauthorized: Invalid or missing Authorization header', 87 + }) 88 + } 89 + 90 + const jwt = authHeader.replace('Bearer ', '').trim() 91 + 92 + try { 93 + // The service DID and resolver should be passed from app context 94 + const serviceDid = c.get('serviceDid') 95 + const didResolver = c.get('didResolver') as DidResolver 96 + 97 + const parsed = await verifyJwt(jwt, serviceDid, null, async (did: string) => { 98 + return didResolver.resolveAtprotoKey(did) 99 + }) 100 + 101 + // Set auth information in the context for route handlers to access 102 + c.set('did', parsed.iss) 103 + c.set('accessJwt', jwt) 104 + 105 + // Check if admin password is valid 106 + if (!adminPassword || !env.ADMIN_PASSWORDS.includes(adminPassword)) { 107 + throw new HTTPException(403, { 108 + message: 'Forbidden: Invalid admin credentials', 109 + }) 110 + } 111 + 112 + c.set('isAdmin', true) 113 + 114 + await next() 115 + } catch (err) { 116 + if (err instanceof HTTPException) { 117 + throw err 118 + } 119 + throw new HTTPException(401, { 120 + message: 'Unauthorized: Invalid JWT token', 121 + }) 122 + } 73 123 }

+61

services/appview/src/db.ts

··· 342 342 generatorSchema.index({ authorDid: 1, createdAt: -1 }) 343 343 generatorSchema.index({ did: 1, createdAt: -1 }) 344 344 345 + export interface TakedownDocument extends Document { 346 + targetUri: string 347 + targetCid: string 348 + reason: string 349 + takenDownBy: string 350 + takenDownAt: string 351 + } 352 + 353 + export const takedownSchema = new Schema<TakedownDocument>({ 354 + targetUri: { type: String, required: true, unique: true, index: true }, 355 + targetCid: { type: String, required: true }, 356 + reason: { type: String, required: true }, 357 + takenDownBy: { type: String, required: true }, 358 + takenDownAt: { type: String, required: true }, 359 + }) 360 + 361 + // Repository takedown schema 362 + export interface RepoTakedownDocument extends Document { 363 + did: string 364 + reason: string 365 + takenDownBy: string 366 + takenDownAt: string 367 + ref: string | null 368 + } 369 + 370 + export const repoTakedownSchema = new Schema<RepoTakedownDocument>({ 371 + did: { type: String, required: true, unique: true, index: true }, 372 + reason: { type: String, required: true }, 373 + takenDownBy: { type: String, required: true }, 374 + takenDownAt: { type: String, required: true }, 375 + ref: { type: String, required: false, default: null }, 376 + }) 377 + 378 + // Blob takedown schema 379 + export interface BlobTakedownDocument extends Document { 380 + did: string 381 + cid: string 382 + reason: string 383 + takenDownBy: string 384 + takenDownAt: string 385 + ref: string | null 386 + } 387 + 388 + export const blobTakedownSchema = new Schema<BlobTakedownDocument>({ 389 + did: { type: String, required: true, index: true }, 390 + cid: { type: String, required: true, index: true }, 391 + reason: { type: String, required: true }, 392 + takenDownBy: { type: String, required: true }, 393 + takenDownAt: { type: String, required: true }, 394 + ref: { type: String, required: false, default: null }, 395 + }) 396 + 397 + // Ensure compound index on did + cid for blob takedowns 398 + blobTakedownSchema.index({ did: 1, cid: 1 }, { unique: true }) 399 + 345 400 export interface DatabaseModels { 346 401 Like: Model<LikeDocument> 347 402 Post: Model<PostDocument> ··· 353 408 Music: Model<MusicDocument> 354 409 Look: Model<LookDocument> 355 410 Generator: Model<GeneratorDocument> 411 + Takedown: Model<TakedownDocument> 412 + RepoTakedown: Model<RepoTakedownDocument> 413 + BlobTakedown: Model<BlobTakedownDocument> 356 414 } 357 415 358 416 export class Database { ··· 373 431 Music: this.connection.model<MusicDocument>('Music', musicSchema), 374 432 Look: this.connection.model<LookDocument>('Look', lookSchema), 375 433 Generator: this.connection.model<GeneratorDocument>('Generator', generatorSchema), 434 + Takedown: this.connection.model<TakedownDocument>('Takedown', takedownSchema), 435 + RepoTakedown: this.connection.model<RepoTakedownDocument>('RepoTakedown', repoTakedownSchema), 436 + BlobTakedown: this.connection.model<BlobTakedownDocument>('BlobTakedown', blobTakedownSchema), 376 437 } 377 438 } 378 439

+17 -18

services/appview/src/env.ts

··· 1 - import dotenv from 'dotenv' 2 - import { cleanEnv, host, port, str, testOnly } from 'envalid' 1 + import * as dotenv from 'dotenv' 2 + import { envStr, envList, envInt } from '@atproto/common' 3 3 4 4 dotenv.config() 5 5 6 - export const env = cleanEnv(process.env, { 7 - NODE_ENV: str({ 8 - devDefault: testOnly('test'), 9 - choices: ['development', 'production', 'test'], 10 - }), 11 - HOST: host({ devDefault: '0.0.0.0' }), 12 - PORT: port({ devDefault: 3000 }), 13 - PUBLIC_URL: str({ devDefault: '' }), 14 - APPVIEW_K256_PRIVATE_KEY_HEX: str({ devDefault: '' }), 15 - SERVICE_DID: str({ devDefault: 'did:web:localhost' }), 6 + export const env = { 7 + NODE_ENV: envStr('NODE_ENV') ?? 'test', 8 + HOST: envStr('HOST') ?? '0.0.0.0', 9 + PORT: envInt('PORT') ?? 3000, 10 + PUBLIC_URL: envStr('PUBLIC_URL') ?? '', 11 + APPVIEW_K256_PRIVATE_KEY_HEX: envStr('APPVIEW_K256_PRIVATE_KEY_HEX') ?? '', 12 + SERVICE_DID: envStr('SERVICE_DID') ?? 'did:web:localhost', 13 + MOD_SERVICE_DID: envStr('MOD_SERVICE_DID') ?? 'did:web:localhost', 14 + ADMIN_PASSWORDS: envList('ADMIN_PASSWORDS') ?? [], 16 15 17 - DB_NAME: str({ devDefault: 'dev' }), 18 - DB_HOST: str({ devDefault: 'localhost' }), 19 - DB_PORT: port({ devDefault: 27017 }), 20 - DB_USER: str({ devDefault: 'mongo' }), 21 - DB_PASSWORD: str({ devDefault: 'mongo' }), 22 - }) 16 + DB_NAME: envStr('DB_NAME') ?? 'dev', 17 + DB_HOST: envStr('DB_HOST') ?? 'localhost', 18 + DB_PORT: envInt('DB_PORT') ?? 27017, 19 + DB_USER: envStr('DB_USER') ?? 'mongo', 20 + DB_PASSWORD: envStr('DB_PASSWORD') ?? 'mongo', 21 + }

+17

services/appview/src/index.ts

··· 19 19 import { createGetFollowersRouter } from './routes/graph/getFollowers.js' 20 20 import { createGetFollowsRouter } from './routes/graph/getFollows.js' 21 21 import wellKnownRouter from './well-known.js' 22 + import { TakedownService } from './services/takedown.js' 23 + import { createTakedownRouter } from './routes/admin/takedowns.js' 24 + import { takedownFilterMiddleware } from './middleware/takedown-filter.js' 22 25 23 26 export type AppContext = { 24 27 db: Database ··· 26 29 resolver: BidirectionalResolver 27 30 serviceDid: string 28 31 didResolver: DidResolver 32 + takedownService: TakedownService 29 33 } 30 34 31 35 export class Server { ··· 45 49 46 50 // Get service DID from environment 47 51 const serviceDid = env.SERVICE_DID 52 + 53 + // Create takedown service 54 + const takedownService = new TakedownService(db) 48 55 49 56 const ctx = { 50 57 db, ··· 52 59 resolver, 53 60 serviceDid, 54 61 didResolver: baseIdResolver.did, 62 + takedownService, 55 63 } 56 64 57 65 const app = new Hono() ··· 64 72 // Type-safe way to set context variables 65 73 c.set('serviceDid', serviceDid) 66 74 c.set('didResolver', baseIdResolver.did) 75 + c.set('takedownService', takedownService) 67 76 await next() 68 77 }) 69 78 79 + // Apply takedown filter middleware to all routes 80 + app.use('*', takedownFilterMiddleware) 81 + 70 82 // TODO: Remove this after getAuthorFeedRouter is properly implemented on frontend 71 83 const feedRouter = createFeedRouter(ctx) 72 84 app.route('/', feedRouter) ··· 77 89 const getFollowersRouter = createGetFollowersRouter(ctx) 78 90 const getFollowsRouter = createGetFollowsRouter(ctx) 79 91 const getAuthorFeedRouter = createGetAuthorFeedRouter(ctx) 92 + 80 93 app.route('/', getPostsRouter) 81 94 app.route('/', getPostThreadRouter) 82 95 app.route('/', getProfileRouter) 83 96 app.route('/', getFollowersRouter) 84 97 app.route('/', getFollowsRouter) 85 98 app.route('/', getAuthorFeedRouter) 99 + 100 + // Create and configure the takedown router 101 + const takedownRouter = createTakedownRouter({ takedownService }) 102 + app.route('/', takedownRouter) 86 103 87 104 app.route('/', wellKnownRouter()) 88 105

+267

services/appview/src/middleware/takedown-filter.ts

··· 1 + import { Context, Next } from 'hono' 2 + import { TakedownService } from '../services/takedown.js' 3 + 4 + /** 5 + * Middleware that filters out taken-down content from responses 6 + * This is meant to be applied to routes that return content 7 + * that might have been taken down by admins 8 + */ 9 + export const takedownFilterMiddleware = async (c: Context, next: Next) => { 10 + // Skip filtering for admin routes and non-content routes 11 + const path = c.req.path 12 + if (path.startsWith('/admin/') || path === '/' || path.includes('favicon') || path.includes('xrpc/com.atproto.admin.updateSubjectStatus')) { 13 + await next() 14 + return 15 + } 16 + 17 + // Call the next middleware/route handler first 18 + await next() 19 + 20 + // Skip filtering if not a JSON response 21 + const contentType = c.res.headers.get('Content-Type') 22 + if (!contentType || !contentType.includes('application/json')) { 23 + return 24 + } 25 + 26 + try { 27 + // Get the takedown service from context 28 + const takedownService = c.get('takedownService') as TakedownService 29 + 30 + // Get the response body 31 + const body = await c.res.json() 32 + 33 + // Process different response formats 34 + if (body.posts && Array.isArray(body.posts)) { 35 + // For post feeds 36 + const filteredPosts = await filterTakenDownItems(body.posts, takedownService, 'uri') 37 + body.posts = filteredPosts 38 + } else if (body.feed && Array.isArray(body.feed)) { 39 + // For general feeds 40 + const filteredFeed = await filterTakenDownItems(body.feed, takedownService, 'post.uri') 41 + body.feed = filteredFeed 42 + } else if (body.thread && body.thread.post) { 43 + // For thread views 44 + const isThreadTakenDown = await takedownService.isTakenDown(body.thread.post.uri) 45 + if (isThreadTakenDown) { 46 + // If the main post is taken down, return empty thread 47 + body.thread = null 48 + } else if (body.thread.replies) { 49 + // Filter replies if they exist 50 + body.thread.replies = await filterReplies(body.thread.replies, takedownService) 51 + } 52 + } 53 + 54 + // If there are user profiles in the response, filter out taken down repositories 55 + if (body.profiles && Array.isArray(body.profiles)) { 56 + const filteredProfiles = await filterTakenDownRepos(body.profiles, takedownService) 57 + body.profiles = filteredProfiles 58 + } else if (body.profile) { 59 + if (body.profile.did) { 60 + const isRepoTakenDown = await takedownService.isRepoTakenDown(body.profile.did) 61 + if (isRepoTakenDown) { 62 + body.profile = null 63 + } 64 + } 65 + } else if (body.did && body.$type && body.$type.includes('profileView')) { 66 + // For direct ProfileViewDetailed objects (so.sprk.actor.getProfile) 67 + const isRepoTakenDown = await takedownService.isRepoTakenDown(body.did) 68 + if (isRepoTakenDown) { 69 + // Return a minimal placeholder object for taken-down profiles 70 + const takenDownProfile = { 71 + $type: body.$type, 72 + did: body.did, 73 + handle: body.handle || 'unavailable', 74 + moderation: { 75 + takenDown: true 76 + } 77 + } 78 + 79 + // Create a new response with the placeholder instead of trying to modify body 80 + c.res = new Response(JSON.stringify(takenDownProfile), { 81 + status: c.res.status, 82 + headers: c.res.headers, 83 + }) 84 + 85 + // Skip the rest of the processing 86 + return 87 + } 88 + } else if (body.subject) { 89 + // For followers/follows response that has a subject profile 90 + if (body.subject.did) { 91 + const isRepoTakenDown = await takedownService.isRepoTakenDown(body.subject.did) 92 + if (isRepoTakenDown) { 93 + // Keep minimal info about the profile but mark it as taken down 94 + body.subject = { 95 + $type: body.subject.$type, 96 + did: body.subject.did, 97 + handle: body.subject.handle || 'unavailable', 98 + moderation: { 99 + takenDown: true 100 + } 101 + } 102 + } 103 + } 104 + 105 + // Also filter any followers/follows list 106 + if (body.followers && Array.isArray(body.followers)) { 107 + body.followers = await filterTakenDownRepos(body.followers, takedownService) 108 + } 109 + 110 + if (body.follows && Array.isArray(body.follows)) { 111 + body.follows = await filterTakenDownRepos(body.follows, takedownService) 112 + } 113 + } 114 + 115 + // Set the filtered response 116 + c.res = new Response(JSON.stringify(body), { 117 + status: c.res.status, 118 + headers: c.res.headers, 119 + }) 120 + } catch (error) { 121 + // In case of error, just continue with the original response 122 + console.error('Error in takedown filter middleware:', error) 123 + } 124 + } 125 + 126 + // Helper function to filter out taken down items 127 + async function filterTakenDownItems( 128 + items: any[], 129 + takedownService: TakedownService, 130 + uriPath: string 131 + ): Promise<any[]> { 132 + if (!items || !Array.isArray(items)) return items 133 + 134 + const filteredItems: any[] = [] 135 + 136 + for (const item of items) { 137 + // Get the URI based on the specified path (handles nested objects) 138 + const uri = uriPath.split('.').reduce((obj, key) => obj && obj[key], item) 139 + 140 + if (uri) { 141 + const isTakenDown = await takedownService.isTakenDown(uri) 142 + 143 + // Check if author's repo is taken down 144 + let isAuthorTakenDown = false 145 + if (item.author?.did || (item.post?.author?.did)) { 146 + const authorDid = item.author?.did || item.post?.author?.did 147 + isAuthorTakenDown = await takedownService.isRepoTakenDown(authorDid) 148 + } 149 + 150 + // Keep the item only if neither the content nor the author is taken down 151 + if (!isTakenDown && !isAuthorTakenDown) { 152 + // Filter out taken down images if the item has embeds 153 + if (item.embed?.images && Array.isArray(item.embed.images)) { 154 + item.embed.images = await filterTakenDownBlobs(item.embed.images, takedownService) 155 + } 156 + 157 + filteredItems.push(item) 158 + } 159 + } else { 160 + // If URI is not found, keep the item 161 + filteredItems.push(item) 162 + } 163 + } 164 + 165 + return filteredItems 166 + } 167 + 168 + // Helper function to filter out taken down repositories 169 + async function filterTakenDownRepos( 170 + profiles: any[], 171 + takedownService: TakedownService 172 + ): Promise<any[]> { 173 + if (!profiles || !Array.isArray(profiles)) return profiles 174 + 175 + const filteredProfiles: any[] = [] 176 + 177 + for (const profile of profiles) { 178 + if (profile.did) { 179 + const isRepoTakenDown = await takedownService.isRepoTakenDown(profile.did) 180 + if (!isRepoTakenDown) { 181 + filteredProfiles.push(profile) 182 + } else { 183 + // For UI consistency, push a minimal placeholder for taken-down profiles 184 + // if they need to be represented in lists (follows, followers, etc.) 185 + filteredProfiles.push({ 186 + $type: profile.$type, 187 + did: profile.did, 188 + handle: profile.handle || 'unavailable', 189 + moderation: { 190 + takenDown: true 191 + } 192 + }) 193 + } 194 + } else { 195 + // If no DID, keep the profile 196 + filteredProfiles.push(profile) 197 + } 198 + } 199 + 200 + return filteredProfiles 201 + } 202 + 203 + // Helper function to filter out taken down blobs/images 204 + async function filterTakenDownBlobs( 205 + images: any[], 206 + takedownService: TakedownService 207 + ): Promise<any[]> { 208 + if (!images || !Array.isArray(images)) return images 209 + 210 + const filteredImages: any[] = [] 211 + 212 + for (const image of images) { 213 + // Check if the image is taken down based on blob CID 214 + if (image.cid && image.did) { 215 + const isBlobTakenDown = await takedownService.isBlobTakenDown(image.did, image.cid) 216 + if (!isBlobTakenDown) { 217 + filteredImages.push(image) 218 + } 219 + } else { 220 + // If no CID or DID, keep the image 221 + filteredImages.push(image) 222 + } 223 + } 224 + 225 + return filteredImages 226 + } 227 + 228 + // Helper function to recursively filter replies in a thread 229 + async function filterReplies( 230 + replies: any[], 231 + takedownService: TakedownService 232 + ): Promise<any[]> { 233 + if (!replies || !Array.isArray(replies)) return replies 234 + 235 + const filteredReplies: any[] = [] 236 + 237 + for (const reply of replies) { 238 + if (reply.post && reply.post.uri) { 239 + const isTakenDown = await takedownService.isTakenDown(reply.post.uri) 240 + 241 + // Check if author's repo is taken down 242 + let isAuthorTakenDown = false 243 + if (reply.post.author?.did) { 244 + isAuthorTakenDown = await takedownService.isRepoTakenDown(reply.post.author.did) 245 + } 246 + 247 + if (!isTakenDown && !isAuthorTakenDown) { 248 + // If this reply has nested replies, filter those too 249 + if (reply.replies && Array.isArray(reply.replies)) { 250 + reply.replies = await filterReplies(reply.replies, takedownService) 251 + } 252 + 253 + // Filter out taken down images in the post 254 + if (reply.post.embed?.images && Array.isArray(reply.post.embed.images)) { 255 + reply.post.embed.images = await filterTakenDownBlobs(reply.post.embed.images, takedownService) 256 + } 257 + 258 + filteredReplies.push(reply) 259 + } 260 + } else { 261 + // If no post or URI, keep the reply 262 + filteredReplies.push(reply) 263 + } 264 + } 265 + 266 + return filteredReplies 267 + }

+320

services/appview/src/routes/admin/takedowns.ts

··· 1 + import { Hono } from 'hono' 2 + import { zValidator } from '@hono/zod-validator' 3 + import { z } from 'zod' 4 + import { HTTPException } from 'hono/http-exception' 5 + import { TakedownService } from '../../services/takedown.js' 6 + import { adminAuthMiddleware } from '../../auth/middleware.js' 7 + 8 + type TakedownContext = { 9 + takedownService: TakedownService 10 + } 11 + 12 + export const createTakedownRouter = (ctx: TakedownContext) => { 13 + const takedownRoutes = new Hono() 14 + const takedownService = ctx.takedownService 15 + 16 + // Apply admin auth middleware to all admin routes 17 + takedownRoutes.use('/admin/*', adminAuthMiddleware) 18 + 19 + // No auth needed for the Ozone integration endpoint as it will use its own auth 20 + // This will be protected by adminAuthMiddleware before processing 21 + 22 + takedownRoutes.post('/admin/takedowns', zValidator('json', z.object({ 23 + targetUri: z.string(), 24 + targetCid: z.string(), 25 + reason: z.string(), 26 + })), async (c) => { 27 + const { targetUri, targetCid, reason } = c.req.valid('json') 28 + const adminDid = c.get('did') 29 + 30 + try { 31 + await takedownService.takedownContent({ 32 + targetUri, 33 + targetCid, 34 + reason, 35 + adminDid, 36 + }) 37 + 38 + return c.json({ success: true }, 201) 39 + } catch (error) { 40 + throw new HTTPException(500, { message: 'Failed to create takedown' }) 41 + } 42 + }) 43 + 44 + // Remove a takedown (record/post) 45 + takedownRoutes.delete('/admin/takedowns/:uri', async (c) => { 46 + const uri = c.req.param('uri') 47 + 48 + try { 49 + const removed = await takedownService.removeTakedown(uri) 50 + 51 + if (!removed) { 52 + return c.json({ success: false, message: 'Takedown not found' }, 404) 53 + } 54 + 55 + return c.json({ success: true }) 56 + } catch (error) { 57 + throw new HTTPException(500, { message: 'Failed to remove takedown' }) 58 + } 59 + }) 60 + 61 + // Create a repo takedown 62 + takedownRoutes.post('/admin/takedowns/repo', zValidator('json', z.object({ 63 + did: z.string(), 64 + reason: z.string(), 65 + ref: z.string().optional(), 66 + })), async (c) => { 67 + const { did, reason, ref } = c.req.valid('json') 68 + const adminDid = c.get('did') 69 + 70 + try { 71 + await takedownService.takedownRepo({ 72 + did, 73 + reason, 74 + adminDid, 75 + ref, 76 + }) 77 + 78 + return c.json({ success: true }, 201) 79 + } catch (error) { 80 + throw new HTTPException(500, { message: 'Failed to create repo takedown' }) 81 + } 82 + }) 83 + 84 + // Remove a repo takedown 85 + takedownRoutes.delete('/admin/takedowns/repo/:did', async (c) => { 86 + const did = c.req.param('did') 87 + 88 + try { 89 + const removed = await takedownService.removeRepoTakedown(did) 90 + 91 + if (!removed) { 92 + return c.json({ success: false, message: 'Repo takedown not found' }, 404) 93 + } 94 + 95 + return c.json({ success: true }) 96 + } catch (error) { 97 + throw new HTTPException(500, { message: 'Failed to remove repo takedown' }) 98 + } 99 + }) 100 + 101 + // Create a blob takedown 102 + takedownRoutes.post('/admin/takedowns/blob', zValidator('json', z.object({ 103 + did: z.string(), 104 + cid: z.string(), 105 + reason: z.string(), 106 + ref: z.string().optional(), 107 + })), async (c) => { 108 + const { did, cid, reason, ref } = c.req.valid('json') 109 + const adminDid = c.get('did') 110 + 111 + try { 112 + await takedownService.takedownBlob({ 113 + did, 114 + cid, 115 + reason, 116 + adminDid, 117 + ref, 118 + }) 119 + 120 + return c.json({ success: true }, 201) 121 + } catch (error) { 122 + throw new HTTPException(500, { message: 'Failed to create blob takedown' }) 123 + } 124 + }) 125 + 126 + // Remove a blob takedown 127 + takedownRoutes.delete('/admin/takedowns/blob/:did/:cid', async (c) => { 128 + const did = c.req.param('did') 129 + const cid = c.req.param('cid') 130 + 131 + try { 132 + const removed = await takedownService.removeBlobTakedown(did, cid) 133 + 134 + if (!removed) { 135 + return c.json({ success: false, message: 'Blob takedown not found' }, 404) 136 + } 137 + 138 + return c.json({ success: true }) 139 + } catch (error) { 140 + throw new HTTPException(500, { message: 'Failed to remove blob takedown' }) 141 + } 142 + }) 143 + 144 + // List takedowns 145 + takedownRoutes.get('/admin/takedowns', zValidator('query', z.object({ 146 + limit: z.string().optional().transform(val => val ? parseInt(val) : 50), 147 + cursor: z.string().optional(), 148 + })), async (c) => { 149 + const { limit, cursor } = c.req.valid('query') 150 + 151 + try { 152 + const result = await takedownService.listTakedowns(limit, cursor) 153 + return c.json(result) 154 + } catch (error) { 155 + throw new HTTPException(500, { message: 'Failed to list takedowns' }) 156 + } 157 + }) 158 + 159 + // List repo takedowns 160 + takedownRoutes.get('/admin/takedowns/repo', zValidator('query', z.object({ 161 + limit: z.string().optional().transform(val => val ? parseInt(val) : 50), 162 + cursor: z.string().optional(), 163 + })), async (c) => { 164 + const { limit, cursor } = c.req.valid('query') 165 + 166 + try { 167 + const result = await takedownService.listRepoTakedowns(limit, cursor) 168 + return c.json(result) 169 + } catch (error) { 170 + throw new HTTPException(500, { message: 'Failed to list repo takedowns' }) 171 + } 172 + }) 173 + 174 + // List blob takedowns 175 + takedownRoutes.get('/admin/takedowns/blob', zValidator('query', z.object({ 176 + limit: z.string().optional().transform(val => val ? parseInt(val) : 50), 177 + cursor: z.string().optional(), 178 + })), async (c) => { 179 + const { limit, cursor } = c.req.valid('query') 180 + 181 + try { 182 + const result = await takedownService.listBlobTakedowns(limit, cursor) 183 + return c.json(result) 184 + } catch (error) { 185 + throw new HTTPException(500, { message: 'Failed to list blob takedowns' }) 186 + } 187 + }) 188 + 189 + // Check if content is taken down 190 + takedownRoutes.get('/admin/takedowns/check/:uri', async (c) => { 191 + const uri = c.req.param('uri') 192 + 193 + try { 194 + const isTakenDown = await takedownService.isTakenDown(uri) 195 + return c.json({ isTakenDown }) 196 + } catch (error) { 197 + throw new HTTPException(500, { message: 'Failed to check takedown status' }) 198 + } 199 + }) 200 + 201 + // Check if repo is taken down 202 + takedownRoutes.get('/admin/takedowns/check/repo/:did', async (c) => { 203 + const did = c.req.param('did') 204 + 205 + try { 206 + const isTakenDown = await takedownService.isRepoTakenDown(did) 207 + return c.json({ isTakenDown }) 208 + } catch (error) { 209 + throw new HTTPException(500, { message: 'Failed to check repo takedown status' }) 210 + } 211 + }) 212 + 213 + // Check if blob is taken down 214 + takedownRoutes.get('/admin/takedowns/check/blob/:did/:cid', async (c) => { 215 + const did = c.req.param('did') 216 + const cid = c.req.param('cid') 217 + 218 + try { 219 + const isTakenDown = await takedownService.isBlobTakenDown(did, cid) 220 + return c.json({ isTakenDown }) 221 + } catch (error) { 222 + throw new HTTPException(500, { message: 'Failed to check blob takedown status' }) 223 + } 224 + }) 225 + 226 + // XRPC endpoint for Ozone integration: com.atproto.admin.updateSubjectStatus 227 + takedownRoutes.post('/xrpc/com.atproto.admin.updateSubjectStatus', adminAuthMiddleware, zValidator('json', z.object({ 228 + subject: z.object({ 229 + $type: z.string(), 230 + did: z.string().optional(), 231 + uri: z.string().optional(), 232 + cid: z.string().optional(), 233 + }), 234 + takedown: z.object({ 235 + applied: z.boolean(), 236 + ref: z.string().optional(), 237 + }), 238 + })), async (c) => { 239 + const { subject, takedown } = c.req.valid('json') 240 + const adminDid = c.get('did') 241 + 242 + try { 243 + // Handle different subject types 244 + if (subject.$type === 'com.atproto.admin.defs#repoRef') { 245 + // Repository (user account) takedown 246 + if (!subject.did) { 247 + throw new HTTPException(400, { message: 'DID is required for repo takedowns' }) 248 + } 249 + 250 + if (takedown.applied) { 251 + // Apply takedown 252 + await takedownService.takedownRepo({ 253 + did: subject.did, 254 + reason: 'Moderation via Ozone', 255 + adminDid, 256 + ref: takedown.ref, 257 + }) 258 + } else { 259 + // Remove takedown 260 + await takedownService.removeRepoTakedown(subject.did) 261 + } 262 + } else if (subject.$type === 'com.atproto.repo.strongRef') { 263 + // Record (post) takedown 264 + if (!subject.uri || !subject.cid) { 265 + throw new HTTPException(400, { message: 'URI and CID are required for record takedowns' }) 266 + } 267 + 268 + if (takedown.applied) { 269 + // Apply takedown 270 + await takedownService.takedownContent({ 271 + targetUri: subject.uri, 272 + targetCid: subject.cid, 273 + reason: 'Moderation via Ozone', 274 + adminDid, 275 + }) 276 + } else { 277 + // Remove takedown 278 + await takedownService.removeTakedown(subject.uri) 279 + } 280 + } else if (subject.$type === 'com.atproto.admin.defs#repoBlobRef') { 281 + // Blob (image/attachment) takedown 282 + if (!subject.did || !subject.cid) { 283 + throw new HTTPException(400, { message: 'DID and CID are required for blob takedowns' }) 284 + } 285 + 286 + if (takedown.applied) { 287 + // Apply takedown 288 + await takedownService.takedownBlob({ 289 + did: subject.did, 290 + cid: subject.cid, 291 + reason: 'Moderation via Ozone', 292 + adminDid, 293 + ref: takedown.ref, 294 + }) 295 + } else { 296 + // Remove takedown 297 + await takedownService.removeBlobTakedown(subject.did, subject.cid) 298 + } 299 + } else { 300 + throw new HTTPException(400, { message: `Unsupported subject type: ${subject.$type}` }) 301 + } 302 + 303 + // Return the response format expected by Ozone 304 + return c.json({ 305 + subject, 306 + takedown: takedown.applied ? { 307 + applied: takedown.applied, 308 + ref: takedown.ref 309 + } : undefined 310 + }) 311 + } catch (error) { 312 + if (error instanceof HTTPException) { 313 + throw error 314 + } 315 + throw new HTTPException(500, { message: 'Failed to update subject status' }) 316 + } 317 + }) 318 + 319 + return takedownRoutes 320 + }

+203

services/appview/src/services/takedown.ts

··· 1 + import { Database } from '../db.js' 2 + 3 + export class TakedownService { 4 + constructor(private db: Database) {} 5 + 6 + async takedownContent(params: { 7 + targetUri: string 8 + targetCid: string 9 + reason: string 10 + adminDid: string 11 + }): Promise<void> { 12 + const { targetUri, targetCid, reason, adminDid } = params 13 + 14 + // Create a takedown record 15 + await this.db.models.Takedown.create({ 16 + targetUri, 17 + targetCid, 18 + reason, 19 + takenDownBy: adminDid, 20 + takenDownAt: new Date().toISOString(), 21 + }) 22 + } 23 + 24 + // Add a method to handle user repo takedowns 25 + async takedownRepo(params: { 26 + did: string 27 + reason: string 28 + adminDid: string 29 + ref?: string 30 + }): Promise<void> { 31 + const { did, reason, adminDid, ref } = params 32 + 33 + // Create a repo takedown record 34 + await this.db.models.RepoTakedown.create({ 35 + did, 36 + reason, 37 + takenDownBy: adminDid, 38 + takenDownAt: new Date().toISOString(), 39 + ref: ref || null, 40 + }) 41 + } 42 + 43 + // Add a method to handle blob takedowns 44 + async takedownBlob(params: { 45 + did: string 46 + cid: string 47 + reason: string 48 + adminDid: string 49 + ref?: string 50 + }): Promise<void> { 51 + const { did, cid, reason, adminDid, ref } = params 52 + 53 + // Create a blob takedown record 54 + await this.db.models.BlobTakedown.create({ 55 + did, 56 + cid, 57 + reason, 58 + takenDownBy: adminDid, 59 + takenDownAt: new Date().toISOString(), 60 + ref: ref || null, 61 + }) 62 + } 63 + 64 + async isTakenDown(uri: string): Promise<boolean> { 65 + const takedown = await this.db.models.Takedown.findOne({ targetUri: uri }) 66 + return !!takedown 67 + } 68 + 69 + // Add a method to check if a repo is taken down 70 + async isRepoTakenDown(did: string): Promise<boolean> { 71 + const takedown = await this.db.models.RepoTakedown.findOne({ did }) 72 + return !!takedown 73 + } 74 + 75 + // Add a method to check if a blob is taken down 76 + async isBlobTakenDown(did: string, cid: string): Promise<boolean> { 77 + const takedown = await this.db.models.BlobTakedown.findOne({ did, cid }) 78 + return !!takedown 79 + } 80 + 81 + async removeTakedown(targetUri: string): Promise<boolean> { 82 + const result = await this.db.models.Takedown.deleteOne({ targetUri }) 83 + return result.deletedCount > 0 84 + } 85 + 86 + // Add a method to remove repo takedown 87 + async removeRepoTakedown(did: string): Promise<boolean> { 88 + const result = await this.db.models.RepoTakedown.deleteOne({ did }) 89 + return result.deletedCount > 0 90 + } 91 + 92 + // Add a method to remove blob takedown 93 + async removeBlobTakedown(did: string, cid: string): Promise<boolean> { 94 + const result = await this.db.models.BlobTakedown.deleteOne({ did, cid }) 95 + return result.deletedCount > 0 96 + } 97 + 98 + async listTakedowns(limit: number = 50, cursor?: string): Promise<{ 99 + takedowns: Array<{ 100 + targetUri: string 101 + targetCid: string 102 + reason: string 103 + takenDownBy: string 104 + takenDownAt: string 105 + }> 106 + cursor?: string 107 + }> { 108 + const query = cursor 109 + ? { targetUri: { $lt: cursor } } 110 + : {} 111 + 112 + const takedowns = await this.db.models.Takedown 113 + .find(query) 114 + .sort({ targetUri: -1 }) 115 + .limit(limit + 1) 116 + .lean() 117 + 118 + const items = takedowns.slice(0, limit) 119 + 120 + return { 121 + takedowns: items.map(t => ({ 122 + targetUri: t.targetUri, 123 + targetCid: t.targetCid, 124 + reason: t.reason, 125 + takenDownBy: t.takenDownBy, 126 + takenDownAt: t.takenDownAt 127 + })), 128 + cursor: takedowns.length > limit ? takedowns[limit - 1].targetUri : undefined 129 + } 130 + } 131 + 132 + // Add a method to list repo takedowns 133 + async listRepoTakedowns(limit: number = 50, cursor?: string): Promise<{ 134 + repoTakedowns: Array<{ 135 + did: string 136 + reason: string 137 + takenDownBy: string 138 + takenDownAt: string 139 + ref: string | null 140 + }> 141 + cursor?: string 142 + }> { 143 + const query = cursor 144 + ? { did: { $lt: cursor } } 145 + : {} 146 + 147 + const takedowns = await this.db.models.RepoTakedown 148 + .find(query) 149 + .sort({ did: -1 }) 150 + .limit(limit + 1) 151 + .lean() 152 + 153 + const items = takedowns.slice(0, limit) 154 + 155 + return { 156 + repoTakedowns: items.map(t => ({ 157 + did: t.did, 158 + reason: t.reason, 159 + takenDownBy: t.takenDownBy, 160 + takenDownAt: t.takenDownAt, 161 + ref: t.ref 162 + })), 163 + cursor: takedowns.length > limit ? takedowns[limit - 1].did : undefined 164 + } 165 + } 166 + 167 + // Add a method to list blob takedowns 168 + async listBlobTakedowns(limit: number = 50, cursor?: string): Promise<{ 169 + blobTakedowns: Array<{ 170 + did: string 171 + cid: string 172 + reason: string 173 + takenDownBy: string 174 + takenDownAt: string 175 + ref: string | null 176 + }> 177 + cursor?: string 178 + }> { 179 + const query = cursor 180 + ? { did: { $lt: cursor } } 181 + : {} 182 + 183 + const takedowns = await this.db.models.BlobTakedown 184 + .find(query) 185 + .sort({ did: -1, cid: -1 }) 186 + .limit(limit + 1) 187 + .lean() 188 + 189 + const items = takedowns.slice(0, limit) 190 + 191 + return { 192 + blobTakedowns: items.map(t => ({ 193 + did: t.did, 194 + cid: t.cid, 195 + reason: t.reason, 196 + takenDownBy: t.takenDownBy, 197 + takenDownAt: t.takenDownAt, 198 + ref: t.ref 199 + })), 200 + cursor: takedowns.length > limit ? takedowns[limit - 1].did : undefined 201 + } 202 + } 203 + }

+11 -17

services/feed-gen/src/utils/env.ts

··· 1 - import dotenv from 'dotenv' 2 - import { cleanEnv, port, str, testOnly } from 'envalid' 1 + import * as dotenv from 'dotenv' 2 + import { envBool, envInt, envList, envStr } from '@atproto/common' 3 3 4 4 dotenv.config() 5 5 6 - export const env = cleanEnv(process.env, { 7 - NODE_ENV: str({ 8 - devDefault: testOnly('test'), 9 - choices: ['development', 'production', 'test'], 10 - }), 6 + export const env = { 7 + NODE_ENV: envStr('NODE_ENV'), 11 8 12 - FEEDGEN_DOMAIN: str({ 13 - default: 'example.com', 14 - desc: 'Domain of the feed generator for did:web' 15 - }), 9 + FEEDGEN_DOMAIN: envStr('FEEDGEN_DOMAIN'), 16 10 17 - DB_NAME: str({ devDefault: 'dev' }), 18 - DB_HOST: str({ devDefault: 'localhost' }), 19 - DB_PORT: port({ devDefault: 27017 }), 20 - DB_USER: str({ devDefault: 'mongo' }), 21 - DB_PASSWORD: str({ devDefault: 'mongo' }), 22 - }) 11 + DB_NAME: envStr('DB_NAME'), 12 + DB_HOST: envStr('DB_HOST'), 13 + DB_PORT: envInt('DB_PORT'), 14 + DB_USER: envStr('DB_USER'), 15 + DB_PASSWORD: envStr('DB_PASSWORD') 16 + }

+64 -1

services/ingester/bun.lock

··· 4 4 "": { 5 5 "name": "ingester", 6 6 "dependencies": { 7 + "@atproto/common": "^0.4.6", 7 8 "@atproto/identity": "^0.4.6", 8 9 "dotenv": "^16.4.7", 9 10 "envalid": "^8.0.0", ··· 22 23 }, 23 24 }, 24 25 "packages": { 25 - "@atproto/common-web": ["@atproto/common-web@0.4.0", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, "sha512-ZYL0P9myHybNgwh/hBY0HaBzqiLR1B5/ie5bJpLQAg0whRzNA28t8/nU2vh99tbsWcAF0LOD29M8++LyENJLNQ=="], 26 + "@atproto/common": ["@atproto/common@0.4.10", "", { "dependencies": { "@atproto/common-web": "^0.4.1", "@ipld/dag-cbor": "^7.0.3", "cbor-x": "^1.5.1", "iso-datestring-validator": "^2.2.2", "multiformats": "^9.9.0", "pino": "^8.21.0" } }, "sha512-/Yxnax3XOhf46jYpe8/6O3ORjTNMB4YCaxx3V1f+FKy6meTm3GNrJwo8d1CBs0UiTiheRiNATOV3u0s3C7Ydaw=="], 27 + 28 + "@atproto/common-web": ["@atproto/common-web@0.4.1", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, "sha512-Ghh+djHYMAUCktLKwr2IuGgtjcwSWGudp+K7+N7KBA9pDDloOXUEY8Agjc5SHSo9B1QIEFkegClU5n+apn2e0w=="], 26 29 27 30 "@atproto/crypto": ["@atproto/crypto@0.4.4", "", { "dependencies": { "@noble/curves": "^1.7.0", "@noble/hashes": "^1.6.1", "uint8arrays": "3.0.0" } }, "sha512-Yq9+crJ7WQl7sxStVpHgie5Z51R05etaK9DLWYG/7bR5T4bhdcIgF6IfklLShtZwLYdVVj+K15s0BqW9a8PSDA=="], 28 31 29 32 "@atproto/identity": ["@atproto/identity@0.4.6", "", { "dependencies": { "@atproto/common-web": "^0.4.0", "@atproto/crypto": "^0.4.4" } }, "sha512-fJq/cIp9MOgHxZfxuyki6mobk0QxRnbts53DstRixlvb5mOoxwttb9Gp6A8u9q49zBsfOmXNTHmP97I9iMHmTQ=="], 30 33 34 + "@cbor-extract/cbor-extract-darwin-arm64": ["@cbor-extract/cbor-extract-darwin-arm64@2.2.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-P7swiOAdF7aSi0H+tHtHtr6zrpF3aAq/W9FXx5HektRvLTM2O89xCyXF3pk7pLc7QpaY7AoaE8UowVf9QBdh3w=="], 35 + 36 + "@cbor-extract/cbor-extract-darwin-x64": ["@cbor-extract/cbor-extract-darwin-x64@2.2.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-1liF6fgowph0JxBbYnAS7ZlqNYLf000Qnj4KjqPNW4GViKrEql2MgZnAsExhY9LSy8dnvA4C0qHEBgPrll0z0w=="], 37 + 38 + "@cbor-extract/cbor-extract-linux-arm": ["@cbor-extract/cbor-extract-linux-arm@2.2.0", "", { "os": "linux", "cpu": "arm" }, "sha512-QeBcBXk964zOytiedMPQNZr7sg0TNavZeuUCD6ON4vEOU/25+pLhNN6EDIKJ9VLTKaZ7K7EaAriyYQ1NQ05s/Q=="], 39 + 40 + "@cbor-extract/cbor-extract-linux-arm64": ["@cbor-extract/cbor-extract-linux-arm64@2.2.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-rQvhNmDuhjTVXSPFLolmQ47/ydGOFXtbR7+wgkSY0bdOxCFept1hvg59uiLPT2fVDuJFuEy16EImo5tE2x3RsQ=="], 41 + 42 + "@cbor-extract/cbor-extract-linux-x64": ["@cbor-extract/cbor-extract-linux-x64@2.2.0", "", { "os": "linux", "cpu": "x64" }, "sha512-cWLAWtT3kNLHSvP4RKDzSTX9o0wvQEEAj4SKvhWuOVZxiDAeQazr9A+PSiRILK1VYMLeDml89ohxCnUNQNQNCw=="], 43 + 44 + "@cbor-extract/cbor-extract-win32-x64": ["@cbor-extract/cbor-extract-win32-x64@2.2.0", "", { "os": "win32", "cpu": "x64" }, "sha512-l2M+Z8DO2vbvADOBNLbbh9y5ST1RY5sqkWOg/58GkUPBYou/cuNZ68SGQ644f1CvZ8kcOxyZtw06+dxWHIoN/w=="], 45 + 46 + "@ipld/dag-cbor": ["@ipld/dag-cbor@7.0.3", "", { "dependencies": { "cborg": "^1.6.0", "multiformats": "^9.5.4" } }, "sha512-1VVh2huHsuohdXC1bGJNE8WR72slZ9XE2T3wbBBq31dm7ZBatmKLLxrB+XAqafxfRFjv08RZmj/W/ZqaM13AuA=="], 47 + 31 48 "@mongodb-js/saslprep": ["@mongodb-js/saslprep@1.2.0", "", { "dependencies": { "sparse-bitfield": "^3.0.3" } }, "sha512-+ywrb0AqkfaYuhHs6LxKWgqbh3I72EpEgESCw37o+9qPx9WTCkgDm2B+eMrwehGtHBWHFU4GXvnSCNiFhhausg=="], 32 49 33 50 "@noble/curves": ["@noble/curves@1.8.1", "", { "dependencies": { "@noble/hashes": "1.7.1" } }, "sha512-warwspo+UYUPep0Q+vtdVB4Ugn8GGQj8iyB3gnRWsztmUHTI3S1nhdiWNsPUGL0vud7JlRRk1XEu7Lq1KGTnMQ=="], ··· 44 61 45 62 "@types/ws": ["@types/ws@8.18.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-8svvI3hMyvN0kKCJMvTJP/x6Y/EoQbepff882wL+Sn5QsXb3etnamgrJq4isrBxSJj5L2AuXcI0+bgkoAXGUJw=="], 46 63 64 + "abort-controller": ["abort-controller@3.0.0", "", { "dependencies": { "event-target-shim": "^5.0.0" } }, "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg=="], 65 + 47 66 "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="], 48 67 68 + "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], 69 + 49 70 "bson": ["bson@6.10.3", "", {}, "sha512-MTxGsqgYTwfshYWTRdmZRC+M7FnG1b4y7RO7p2k3X24Wq0yv1m77Wsj0BzlPzd/IowgESfsruQCUToa7vbOpPQ=="], 71 + 72 + "buffer": ["buffer@6.0.3", "", { "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" } }, "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA=="], 50 73 51 74 "bun-types": ["bun-types@1.2.5", "", { "dependencies": { "@types/node": "*", "@types/ws": "~8.5.10" } }, "sha512-3oO6LVGGRRKI4kHINx5PIdIgnLRb7l/SprhzqXapmoYkFl5m4j6EvALvbDVuuBFaamB46Ap6HCUxIXNLCGy+tg=="], 52 75 76 + "cbor-extract": ["cbor-extract@2.2.0", "", { "dependencies": { "node-gyp-build-optional-packages": "5.1.1" }, "optionalDependencies": { "@cbor-extract/cbor-extract-darwin-arm64": "2.2.0", "@cbor-extract/cbor-extract-darwin-x64": "2.2.0", "@cbor-extract/cbor-extract-linux-arm": "2.2.0", "@cbor-extract/cbor-extract-linux-arm64": "2.2.0", "@cbor-extract/cbor-extract-linux-x64": "2.2.0", "@cbor-extract/cbor-extract-win32-x64": "2.2.0" }, "bin": { "download-cbor-prebuilds": "bin/download-prebuilds.js" } }, "sha512-Ig1zM66BjLfTXpNgKpvBePq271BPOvu8MR0Jl080yG7Jsl+wAZunfrwiwA+9ruzm/WEdIV5QF/bjDZTqyAIVHA=="], 77 + 78 + "cbor-x": ["cbor-x@1.6.0", "", { "optionalDependencies": { "cbor-extract": "^2.2.0" } }, "sha512-0kareyRwHSkL6ws5VXHEf8uY1liitysCVJjlmhaLG+IXLqhSaOO+t63coaso7yjwEzWZzLy8fJo06gZDVQM9Qg=="], 79 + 80 + "cborg": ["cborg@1.10.2", "", { "bin": { "cborg": "cli.js" } }, "sha512-b3tFPA9pUr2zCUiCfRd2+wok2/LBSNUMKOuRRok+WlvvAgEt/PlbgPTsZUcwCOs53IJvLgTp0eotwtosE6njug=="], 81 + 53 82 "colorette": ["colorette@2.0.20", "", {}, "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w=="], 54 83 55 84 "dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="], 56 85 57 86 "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="], 87 + 88 + "detect-libc": ["detect-libc@2.0.3", "", {}, "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw=="], 58 89 59 90 "dotenv": ["dotenv@16.4.7", "", {}, "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ=="], 60 91 ··· 62 93 63 94 "envalid": ["envalid@8.0.0", "", { "dependencies": { "tslib": "2.6.2" } }, "sha512-PGeYJnJB5naN0ME6SH8nFcDj9HVbLpYIfg1p5lAyM9T4cH2lwtu2fLbozC/bq+HUUOIFxhX/LP0/GmlqPHT4tQ=="], 64 95 96 + "event-target-shim": ["event-target-shim@5.0.1", "", {}, "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="], 97 + 98 + "events": ["events@3.3.0", "", {}, "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="], 99 + 65 100 "fast-copy": ["fast-copy@3.0.2", "", {}, "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="], 66 101 67 102 "fast-redact": ["fast-redact@3.5.0", "", {}, "sha512-dwsoQlS7h9hMeYUq1W++23NDcBLV4KqONnITDV9DjfS3q1SgDGVrBdvvTLUotWtPSD7asWDV9/CmsZPy8Hf70A=="], ··· 71 106 "graphemer": ["graphemer@1.4.0", "", {}, "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag=="], 72 107 73 108 "help-me": ["help-me@5.0.0", "", {}, "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg=="], 109 + 110 + "ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="], 111 + 112 + "iso-datestring-validator": ["iso-datestring-validator@2.2.2", "", {}, "sha512-yLEMkBbLZTlVQqOnQ4FiMujR6T4DEcCb1xizmvXS+OxuhwcbtynoosRzdMA69zZCShCNAbi+gJ71FxZBBXx1SA=="], 74 113 75 114 "joycon": ["joycon@3.1.1", "", {}, "sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw=="], 76 115 ··· 94 133 95 134 "multiformats": ["multiformats@9.9.0", "", {}, "sha512-HoMUjhH9T8DDBNT+6xzkrd9ga/XiBI4xLr58LJACwK6G3HTOPeMz4nB4KJs33L2BelrIJa7P0VuNaVF3hMYfjg=="], 96 135 136 + "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.1.1", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-test": "build-test.js", "node-gyp-build-optional-packages-optional": "optional.js" } }, "sha512-+P72GAjVAbTxjjwUmwjVrqrdZROD4nf8KgpBoDxqXXTiYZZt/ud60dE5yvCSr9lRO8e8yv6kgJIC0K0PfZFVQw=="], 137 + 97 138 "on-exit-leak-free": ["on-exit-leak-free@2.1.2", "", {}, "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA=="], 98 139 99 140 "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], ··· 105 146 "pino-pretty": ["pino-pretty@13.0.0", "", { "dependencies": { "colorette": "^2.0.7", "dateformat": "^4.6.3", "fast-copy": "^3.0.2", "fast-safe-stringify": "^2.1.1", "help-me": "^5.0.0", "joycon": "^3.1.1", "minimist": "^1.2.6", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^2.0.0", "pump": "^3.0.0", "secure-json-parse": "^2.4.0", "sonic-boom": "^4.0.1", "strip-json-comments": "^3.1.1" }, "bin": { "pino-pretty": "bin.js" } }, "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA=="], 106 147 107 148 "pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="], 149 + 150 + "process": ["process@0.11.10", "", {}, "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A=="], 108 151 109 152 "process-warning": ["process-warning@4.0.1", "", {}, "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q=="], 110 153 ··· 114 157 115 158 "quick-format-unescaped": ["quick-format-unescaped@4.0.4", "", {}, "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg=="], 116 159 160 + "readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], 161 + 117 162 "real-require": ["real-require@0.2.0", "", {}, "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg=="], 163 + 164 + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], 118 165 119 166 "safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="], 120 167 ··· 128 175 129 176 "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], 130 177 178 + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], 179 + 131 180 "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], 132 181 133 182 "thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="], ··· 152 201 153 202 "zod": ["zod@3.24.2", "", {}, "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ=="], 154 203 204 + "@atproto/common/pino": ["pino@8.21.0", "", { "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", "pino-abstract-transport": "^1.2.0", "pino-std-serializers": "^6.0.0", "process-warning": "^3.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", "sonic-boom": "^3.7.0", "thread-stream": "^2.6.0" }, "bin": { "pino": "bin.js" } }, "sha512-ip4qdzjkAyDDZklUaZkcRFb2iA118H9SgRh8yzTkSQK8HilsOJF7rSY8HoW5+I0M46AZgX/pxbprf2vvzQCE0Q=="], 205 + 206 + "@atproto/identity/@atproto/common-web": ["@atproto/common-web@0.4.0", "", { "dependencies": { "graphemer": "^1.4.0", "multiformats": "^9.9.0", "uint8arrays": "3.0.0", "zod": "^3.23.8" } }, "sha512-ZYL0P9myHybNgwh/hBY0HaBzqiLR1B5/ie5bJpLQAg0whRzNA28t8/nU2vh99tbsWcAF0LOD29M8++LyENJLNQ=="], 207 + 155 208 "bun-types/@types/ws": ["@types/ws@8.5.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw=="], 209 + 210 + "@atproto/common/pino/pino-abstract-transport": ["pino-abstract-transport@1.2.0", "", { "dependencies": { "readable-stream": "^4.0.0", "split2": "^4.0.0" } }, "sha512-Guhh8EZfPCfH+PMXAb6rKOjGQEoy0xlAIn+irODG5kgfYV+BQ0rGYYWTIel3P5mmyXqkYkPmdIkywsn6QKUR1Q=="], 211 + 212 + "@atproto/common/pino/pino-std-serializers": ["pino-std-serializers@6.2.2", "", {}, "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="], 213 + 214 + "@atproto/common/pino/process-warning": ["process-warning@3.0.0", "", {}, "sha512-mqn0kFRl0EoqhnL0GQ0veqFHyIN1yig9RHh/InzORTUiZHFRAur+aMtRkELNwGs9aNwKS6tg/An4NYBPGwvtzQ=="], 215 + 216 + "@atproto/common/pino/sonic-boom": ["sonic-boom@3.8.1", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-y4Z8LCDBuum+PBP3lSV7RHrXscqksve/bi0as7mhwVnBW+/wUqKT/2Kb7um8yqcFy0duYbbPxzt89Zy2nOCaxg=="], 217 + 218 + "@atproto/common/pino/thread-stream": ["thread-stream@2.7.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-qQiRWsU/wvNolI6tbbCKd9iKaTnCXsTwVxhhKM6nctPdujTyztjlbUkUTUymidWcMnZ5pWR0ej4a0tjsW021vw=="], 156 219 } 157 220 }

+1

services/ingester/package.json

··· 12 12 "@types/ws": "^8.18.0" 13 13 }, 14 14 "dependencies": { 15 + "@atproto/common": "^0.4.6", 15 16 "@atproto/identity": "^0.4.6", 16 17 "dotenv": "^16.4.7", 17 18 "envalid": "^8.0.0",

+13 -17

services/ingester/src/utils/env.ts

··· 1 - import dotenv from 'dotenv' 2 - import { cleanEnv, host, port, str, testOnly } from 'envalid' 1 + import * as dotenv from 'dotenv' 2 + import { envBool, envInt, envList, envStr } from '@atproto/common' 3 3 4 4 dotenv.config() 5 5 6 - export const env = cleanEnv(process.env, { 7 - NODE_ENV: str({ 8 - devDefault: testOnly('test'), 9 - choices: ['development', 'production', 'test'], 10 - }), 6 + export const env = { 7 + NODE_ENV: envStr('NODE_ENV') ?? 'test', 11 8 12 - JETSTREAM_URL: str({ 13 - default: 'wss://jetstream2.us-east.bsky.network/subscribe', 14 - desc: 'Websocket URL for Jetstream connection' 15 - }), 9 + JETSTREAM_URL: 10 + envStr('JETSTREAM_URL') ?? 11 + 'wss://jetstream2.us-east.bsky.network/subscribe', 16 12 17 - DB_NAME: str({ devDefault: 'dev' }), 18 - DB_HOST: str({ devDefault: 'localhost' }), 19 - DB_PORT: port({ devDefault: 27017 }), 20 - DB_USER: str({ devDefault: 'mongo' }), 21 - DB_PASSWORD: str({ devDefault: 'mongo' }), 22 - }) 13 + DB_NAME: envStr('DB_NAME') ?? 'dev', 14 + DB_HOST: envStr('DB_HOST') ?? 'localhost', 15 + DB_PORT: envInt('DB_PORT') ?? 27017, 16 + DB_USER: envStr('DB_USER') ?? 'mongo', 17 + DB_PASSWORD: envStr('DB_PASSWORD') ?? 'mongo', 18 + }

Configure Feed

Configure Feed