[READ ONLY MIRROR] Spark Social AppView Server
github.com/sprksocial/server
atproto
deno
hono
lexicon
fix takedown repo
+189
-82
+2
services/appview/package.json
+2
services/appview/package.json
···
30
30
"express": "^4.21.2",
31
31
"hono": "^4.7.4",
32
32
"iron-session": "^8.0.4",
33
+
"lodash": "^4.17.21",
33
34
"mongoose": "^8.12.1",
34
35
"multiformats": "^9.9.0",
35
36
"pino": "^9.6.0",
···
40
41
"devDependencies": {
41
42
"@atproto/lex-cli": "^0.6.2",
42
43
"@types/express": "^5.0.0",
44
+
"@types/lodash": "^4.17.16",
43
45
"@types/node": "^22.13.10",
44
46
"@types/pg": "^8.11.11",
45
47
"@types/ws": "^8.18.0",
+14
services/appview/pnpm-lock.yaml
+14
services/appview/pnpm-lock.yaml
···
60
60
iron-session:
61
61
specifier: ^8.0.4
62
62
version: 8.0.4
63
+
lodash:
64
+
specifier: ^4.17.21
65
+
version: 4.17.21
63
66
mongoose:
64
67
specifier: ^8.12.1
65
68
version: 8.12.1
···
86
89
'@types/express':
87
90
specifier: ^5.0.0
88
91
version: 5.0.0
92
+
'@types/lodash':
93
+
specifier: ^4.17.16
94
+
version: 4.17.16
89
95
'@types/node':
90
96
specifier: ^22.13.10
91
97
version: 22.13.10
···
858
864
859
865
/@types/json-schema@7.0.15:
860
866
resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==}
867
+
dev: true
868
+
869
+
/@types/lodash@4.17.16:
870
+
resolution: {integrity: sha512-HX7Em5NYQAXKW+1T+FiuG27NGwzJfCX3s1GjOa7ujxZa52kjJLOr4FUxT+giF6Tgxv1e+/czV/iTtBw27WTU9g==}
861
871
dev: true
862
872
863
873
/@types/mime@1.3.5:
···
1894
1904
/lodash.merge@4.6.2:
1895
1905
resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==}
1896
1906
dev: true
1907
+
1908
+
/lodash@4.17.21:
1909
+
resolution: {integrity: sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==}
1910
+
dev: false
1897
1911
1898
1912
/lru-cache@10.4.3:
1899
1913
resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}
+173
-82
services/appview/src/middleware/takedown-filter.ts
+173
-82
services/appview/src/middleware/takedown-filter.ts
···
1
1
import { Context, Next } from 'hono'
2
2
import { TakedownService } from '../services/takedown.js'
3
+
import { get } from 'lodash'
3
4
4
5
/**
5
6
* Middleware that filters out taken-down content from responses
···
13
14
await next()
14
15
return
15
16
}
16
-
17
+
17
18
// Call the next middleware/route handler first
18
19
await next()
19
-
20
+
20
21
// Skip filtering if not a JSON response
21
22
const contentType = c.res.headers.get('Content-Type')
22
23
if (!contentType || !contentType.includes('application/json')) {
23
24
return
24
25
}
25
-
26
+
26
27
try {
27
28
// Get the takedown service from context
28
29
const takedownService = c.get('takedownService') as TakedownService
29
-
30
-
// Get the response body
30
+
31
31
const body = await c.res.json()
32
-
33
-
// Process different response formats
32
+
33
+
const targetDid = body.did || body.user?.did || body.actor?.did || body.profile?.did || body.subject?.did
34
+
if (targetDid) {
35
+
const isRepoTakenDown = await takedownService.isRepoTakenDown(targetDid)
36
+
if (isRepoTakenDown) {
37
+
// For specific user/profile views, return minimal placeholder
38
+
if (body.did && body.$type && body.$type.includes('profileView')) {
39
+
const takenDownProfile = {
40
+
$type: body.$type,
41
+
did: body.did,
42
+
handle: body.handle || 'unavailable',
43
+
moderation: {
44
+
takenDown: true,
45
+
},
46
+
}
47
+
c.res = new Response(JSON.stringify(takenDownProfile), {
48
+
status: c.res.status,
49
+
headers: c.res.headers,
50
+
})
51
+
return
52
+
} else {
53
+
// For other single-user responses, null out or minimize the content
54
+
c.res = new Response(JSON.stringify({
55
+
error: 'Content unavailable - repository has been taken down',
56
+
code: 404
57
+
}), {
58
+
status: 404,
59
+
headers: c.res.headers,
60
+
})
61
+
return
62
+
}
63
+
}
64
+
}
65
+
66
+
// Continue with specific content type filtering
34
67
if (body.posts && Array.isArray(body.posts)) {
35
-
// For post feeds
36
-
const filteredPosts = await filterTakenDownItems(body.posts, takedownService, 'uri')
68
+
const filteredPosts = await filterTakenDownItems(
69
+
body.posts,
70
+
takedownService,
71
+
'uri',
72
+
)
37
73
body.posts = filteredPosts
38
74
} else if (body.feed && Array.isArray(body.feed)) {
39
-
// For general feeds
40
-
const filteredFeed = await filterTakenDownItems(body.feed, takedownService, 'post.uri')
75
+
const filteredFeed = await filterTakenDownItems(
76
+
body.feed,
77
+
takedownService,
78
+
'post.uri',
79
+
)
41
80
body.feed = filteredFeed
42
81
} else if (body.thread && body.thread.post) {
43
-
// For thread views
44
-
const isThreadTakenDown = await takedownService.isTakenDown(body.thread.post.uri)
45
-
if (isThreadTakenDown) {
46
-
// If the main post is taken down, return empty thread
82
+
const isThreadTakenDown = await takedownService.isTakenDown(
83
+
body.thread.post.uri,
84
+
)
85
+
86
+
// Also check if the thread author repo is taken down
87
+
let isAuthorTakenDown = false
88
+
if (body.thread.post.author?.did) {
89
+
isAuthorTakenDown = await takedownService.isRepoTakenDown(
90
+
body.thread.post.author.did
91
+
)
92
+
}
93
+
94
+
if (isThreadTakenDown || isAuthorTakenDown) {
47
95
body.thread = null
48
96
} else if (body.thread.replies) {
49
-
// Filter replies if they exist
50
-
body.thread.replies = await filterReplies(body.thread.replies, takedownService)
97
+
body.thread.replies = await filterReplies(
98
+
body.thread.replies,
99
+
takedownService,
100
+
)
51
101
}
52
102
}
53
103
54
104
// If there are user profiles in the response, filter out taken down repositories
55
105
if (body.profiles && Array.isArray(body.profiles)) {
56
-
const filteredProfiles = await filterTakenDownRepos(body.profiles, takedownService)
106
+
const filteredProfiles = await filterTakenDownRepos(
107
+
body.profiles,
108
+
takedownService,
109
+
)
57
110
body.profiles = filteredProfiles
58
111
} else if (body.profile) {
59
112
if (body.profile.did) {
60
-
const isRepoTakenDown = await takedownService.isRepoTakenDown(body.profile.did)
113
+
const isRepoTakenDown = await takedownService.isRepoTakenDown(
114
+
body.profile.did,
115
+
)
61
116
if (isRepoTakenDown) {
62
117
body.profile = null
63
118
}
···
72
127
did: body.did,
73
128
handle: body.handle || 'unavailable',
74
129
moderation: {
75
-
takenDown: true
76
-
}
130
+
takenDown: true,
131
+
},
77
132
}
78
-
133
+
79
134
// Create a new response with the placeholder instead of trying to modify body
80
135
c.res = new Response(JSON.stringify(takenDownProfile), {
81
136
status: c.res.status,
82
137
headers: c.res.headers,
83
138
})
84
-
139
+
85
140
// Skip the rest of the processing
86
141
return
87
142
}
88
143
} else if (body.subject) {
89
144
// For followers/follows response that has a subject profile
90
145
if (body.subject.did) {
91
-
const isRepoTakenDown = await takedownService.isRepoTakenDown(body.subject.did)
146
+
const isRepoTakenDown = await takedownService.isRepoTakenDown(
147
+
body.subject.did,
148
+
)
92
149
if (isRepoTakenDown) {
93
150
// Keep minimal info about the profile but mark it as taken down
94
151
body.subject = {
···
96
153
did: body.subject.did,
97
154
handle: body.subject.handle || 'unavailable',
98
155
moderation: {
99
-
takenDown: true
100
-
}
156
+
takenDown: true,
157
+
},
101
158
}
102
159
}
103
160
}
104
-
161
+
105
162
// Also filter any followers/follows list
106
163
if (body.followers && Array.isArray(body.followers)) {
107
-
body.followers = await filterTakenDownRepos(body.followers, takedownService)
164
+
body.followers = await filterTakenDownRepos(
165
+
body.followers,
166
+
takedownService,
167
+
)
108
168
}
109
-
169
+
110
170
if (body.follows && Array.isArray(body.follows)) {
111
171
body.follows = await filterTakenDownRepos(body.follows, takedownService)
112
172
}
113
173
}
114
-
174
+
115
175
// Set the filtered response
116
176
c.res = new Response(JSON.stringify(body), {
117
177
status: c.res.status,
···
125
185
126
186
// Helper function to filter out taken down items
127
187
async function filterTakenDownItems(
128
-
items: any[],
188
+
items: Record<string, any>[],
129
189
takedownService: TakedownService,
130
-
uriPath: string
131
-
): Promise<any[]> {
132
-
if (!items || !Array.isArray(items)) return items
133
-
134
-
const filteredItems: any[] = []
135
-
190
+
uriPath: string,
191
+
) {
192
+
if (!items || items.length === 0) {
193
+
return items
194
+
}
195
+
196
+
const filteredItems: Record<string, any>[] = []
197
+
136
198
for (const item of items) {
137
-
// Get the URI based on the specified path (handles nested objects)
138
-
const uri = uriPath.split('.').reduce((obj, key) => obj && obj[key], item)
139
-
199
+
let isTakenDown = false
200
+
201
+
// Get URI for this specific content
202
+
const uri = get(item, uriPath) as string | undefined
140
203
if (uri) {
141
-
const isTakenDown = await takedownService.isTakenDown(uri)
142
-
143
-
// Check if author's repo is taken down
144
-
let isAuthorTakenDown = false
145
-
if (item.author?.did || (item.post?.author?.did)) {
146
-
const authorDid = item.author?.did || item.post?.author?.did
147
-
isAuthorTakenDown = await takedownService.isRepoTakenDown(authorDid)
148
-
}
149
-
150
-
// Keep the item only if neither the content nor the author is taken down
151
-
if (!isTakenDown && !isAuthorTakenDown) {
152
-
// Filter out taken down images if the item has embeds
153
-
if (item.embed?.images && Array.isArray(item.embed.images)) {
154
-
item.embed.images = await filterTakenDownBlobs(item.embed.images, takedownService)
204
+
isTakenDown = await takedownService.isTakenDown(uri)
205
+
}
206
+
207
+
// Check if author's repo is taken down
208
+
let isAuthorTakenDown = false
209
+
// Look for author DID in common locations
210
+
const authorDid = get(item, 'author.did') ||
211
+
get(item, 'post.author.did') ||
212
+
get(item, 'user.did') ||
213
+
get(item, 'actor.did')
214
+
215
+
if (authorDid) {
216
+
isAuthorTakenDown = await takedownService.isRepoTakenDown(authorDid)
217
+
}
218
+
219
+
// Keep the item only if neither the content nor the author is taken down
220
+
if (!isTakenDown && !isAuthorTakenDown) {
221
+
// Also check for any embedded items like quotes or replies
222
+
if (item.embed && item.embed.record && item.embed.record.author?.did) {
223
+
const embedAuthorTakenDown = await takedownService.isRepoTakenDown(item.embed.record.author.did)
224
+
if (embedAuthorTakenDown) {
225
+
// Null out the embed if from a taken-down repo
226
+
item.embed = {
227
+
$type: item.embed.$type,
228
+
takenDown: true
229
+
}
230
+
} else if (item.embed.record.uri) {
231
+
// Check if the specific embedded content is taken down
232
+
const embedContentTakenDown = await takedownService.isTakenDown(item.embed.record.uri)
233
+
if (embedContentTakenDown) {
234
+
item.embed = {
235
+
$type: item.embed.$type,
236
+
takenDown: true
237
+
}
238
+
}
155
239
}
156
-
157
-
filteredItems.push(item)
158
240
}
159
-
} else {
160
-
// If URI is not found, keep the item
161
241
filteredItems.push(item)
162
242
}
163
243
}
164
-
244
+
165
245
return filteredItems
166
246
}
167
247
168
248
// Helper function to filter out taken down repositories
169
249
async function filterTakenDownRepos(
170
250
profiles: any[],
171
-
takedownService: TakedownService
251
+
takedownService: TakedownService,
172
252
): Promise<any[]> {
173
253
if (!profiles || !Array.isArray(profiles)) return profiles
174
-
254
+
175
255
const filteredProfiles: any[] = []
176
-
256
+
177
257
for (const profile of profiles) {
178
258
if (profile.did) {
179
259
const isRepoTakenDown = await takedownService.isRepoTakenDown(profile.did)
···
187
267
did: profile.did,
188
268
handle: profile.handle || 'unavailable',
189
269
moderation: {
190
-
takenDown: true
191
-
}
270
+
takenDown: true,
271
+
},
192
272
})
193
273
}
194
274
} else {
···
196
276
filteredProfiles.push(profile)
197
277
}
198
278
}
199
-
279
+
200
280
return filteredProfiles
201
281
}
202
282
203
283
// Helper function to filter out taken down blobs/images
204
284
async function filterTakenDownBlobs(
205
285
images: any[],
206
-
takedownService: TakedownService
286
+
takedownService: TakedownService,
207
287
): Promise<any[]> {
208
288
if (!images || !Array.isArray(images)) return images
209
-
289
+
210
290
const filteredImages: any[] = []
211
-
291
+
212
292
for (const image of images) {
213
293
// Check if the image is taken down based on blob CID
214
294
if (image.cid && image.did) {
215
-
const isBlobTakenDown = await takedownService.isBlobTakenDown(image.did, image.cid)
295
+
const isBlobTakenDown = await takedownService.isBlobTakenDown(
296
+
image.did,
297
+
image.cid,
298
+
)
216
299
if (!isBlobTakenDown) {
217
300
filteredImages.push(image)
218
301
}
···
221
304
filteredImages.push(image)
222
305
}
223
306
}
224
-
307
+
225
308
return filteredImages
226
309
}
227
310
228
311
// Helper function to recursively filter replies in a thread
229
312
async function filterReplies(
230
313
replies: any[],
231
-
takedownService: TakedownService
314
+
takedownService: TakedownService,
232
315
): Promise<any[]> {
233
316
if (!replies || !Array.isArray(replies)) return replies
234
-
317
+
235
318
const filteredReplies: any[] = []
236
-
319
+
237
320
for (const reply of replies) {
238
321
if (reply.post && reply.post.uri) {
239
322
const isTakenDown = await takedownService.isTakenDown(reply.post.uri)
240
-
323
+
241
324
// Check if author's repo is taken down
242
325
let isAuthorTakenDown = false
243
326
if (reply.post.author?.did) {
244
-
isAuthorTakenDown = await takedownService.isRepoTakenDown(reply.post.author.did)
327
+
isAuthorTakenDown = await takedownService.isRepoTakenDown(
328
+
reply.post.author.did,
329
+
)
245
330
}
246
-
331
+
247
332
if (!isTakenDown && !isAuthorTakenDown) {
248
333
// If this reply has nested replies, filter those too
249
334
if (reply.replies && Array.isArray(reply.replies)) {
250
335
reply.replies = await filterReplies(reply.replies, takedownService)
251
336
}
252
-
337
+
253
338
// Filter out taken down images in the post
254
-
if (reply.post.embed?.images && Array.isArray(reply.post.embed.images)) {
255
-
reply.post.embed.images = await filterTakenDownBlobs(reply.post.embed.images, takedownService)
339
+
if (
340
+
reply.post.embed?.images &&
341
+
Array.isArray(reply.post.embed.images)
342
+
) {
343
+
reply.post.embed.images = await filterTakenDownBlobs(
344
+
reply.post.embed.images,
345
+
takedownService,
346
+
)
256
347
}
257
-
348
+
258
349
filteredReplies.push(reply)
259
350
}
260
351
} else {
···
262
353
filteredReplies.push(reply)
263
354
}
264
355
}
265
-
356
+
266
357
return filteredReplies
267
-
}
358
+
}