[READ ONLY MIRROR] Spark Social AppView Server
github.com/sprksocial/server
atproto
deno
hono
lexicon
graceful failing for indexing (#47)
* graceful failing for indexing
* follow use set...
* all in on optimistic findOneAndUpdate
authored by
Roscoe Rubin-Rottenberg
and committed by
GitHub
2773526b
0636329f
+451
-1021
+5
-5
data-plane/indexing/index.ts
+5
-5
data-plane/indexing/index.ts
···
26
26
import * as Story from "./plugins/story.ts";
27
27
import * as Audio from "./plugins/audio.ts";
28
28
import { RecordProcessor } from "./processor.ts";
29
-
import { Logger } from "@logtape/logtape";
29
+
import { getLogger, Logger } from "@logtape/logtape";
30
30
import { ServerConfig } from "../../config.ts";
31
31
32
32
export class IndexingService {
···
42
42
story: Story.PluginType;
43
43
audio: Audio.PluginType;
44
44
};
45
+
logger: Logger;
45
46
46
47
constructor(
47
48
public db: Database,
48
49
public cfg: ServerConfig,
49
50
public idResolver: IdResolver,
50
51
public background: BackgroundQueue,
51
-
public logger: Logger,
52
52
) {
53
+
this.logger = getLogger(["appview", "indexer"]);
53
54
this.records = {
54
55
post: Post.makePlugin(this.db, this.background),
55
56
reply: Reply.makePlugin(this.db, this.background),
···
70
71
this.cfg,
71
72
this.idResolver,
72
73
this.background,
73
-
this.logger,
74
74
);
75
75
}
76
76
···
157
157
158
158
const actorExists = await this.db.models.Actor.findOne({ did }).lean();
159
159
if (!actorExists) {
160
-
console.log(
160
+
this.logger.info(
161
161
`indexRepo: No actor record found for ${did}, indexing handle first`,
162
162
);
163
163
await this.indexHandle(did, now);
···
179
179
const repoRecords = formatCheckout(did, verifiedRepo);
180
180
const diff = findDiffFromCheckout(currRecords, repoRecords);
181
181
182
-
console.log(`Indexing ${diff.length} records for ${did}:`);
182
+
this.logger.info(`Indexing ${diff.length} records for ${did}:`);
183
183
184
184
await Promise.all(
185
185
diff.map(async (op) => {
+6
-15
data-plane/indexing/plugins/audio.ts
+6
-15
data-plane/indexing/plugins/audio.ts
···
31
31
};
32
32
33
33
// Use findOneAndUpdate with upsert to handle potential duplicate key errors
34
-
try {
35
-
const insertedAudio = await db.models.Audio.findOneAndUpdate(
36
-
{ uri: audio.uri },
37
-
audio,
38
-
{ upsert: true, new: true },
39
-
);
40
-
return insertedAudio;
41
-
} catch (err) {
42
-
// Handle duplicate key errors gracefully
43
-
const mongoError = err as { code?: number };
44
-
if (mongoError.code === 11000) {
45
-
return null; // Silently skip duplicates
46
-
}
47
-
throw err;
48
-
}
34
+
const insertedAudio = await db.models.Audio.findOneAndUpdate(
35
+
{ uri: audio.uri },
36
+
{ $set: audio },
37
+
{ upsert: true, new: true },
38
+
);
39
+
return insertedAudio;
49
40
};
50
41
51
42
const findDuplicate = (): AtUri | null => {
+7
-29
data-plane/indexing/plugins/block.ts
+7
-29
data-plane/indexing/plugins/block.ts
···
26
26
indexedAt: timestamp,
27
27
};
28
28
29
-
// Check if block already exists by URI
30
-
const existingBlockByUri = await db.models.Block.findOne({ uri: block.uri })
31
-
.lean();
32
-
if (existingBlockByUri) {
33
-
return null; // Block already indexed
34
-
}
35
-
36
-
// Check if a block with same authorDid+subject already exists
37
-
const existingBlockByComposite = await db.models.Block.findOne({
38
-
authorDid: block.authorDid,
39
-
subject: block.subject,
40
-
}).lean();
41
-
if (existingBlockByComposite) {
42
-
return null; // Block with same author+subject already exists
43
-
}
44
-
45
-
// Insert the new block
46
-
try {
47
-
const insertedBlock = new db.models.Block(block);
48
-
await insertedBlock.save();
49
-
return insertedBlock;
50
-
} catch (err) {
51
-
// Handle duplicate key errors gracefully
52
-
const mongoError = err as { code?: number };
53
-
if (mongoError.code === 11000) {
54
-
return null; // Silently skip duplicates
55
-
}
56
-
throw err;
57
-
}
29
+
// Use findOneAndUpdate with upsert to handle duplicates gracefully
30
+
const insertedBlock = await db.models.Block.findOneAndUpdate(
31
+
{ uri: block.uri },
32
+
{ $set: block },
33
+
{ upsert: true, new: true },
34
+
);
35
+
return insertedBlock;
58
36
};
59
37
60
38
const findDuplicate = async (
+37
-54
data-plane/indexing/plugins/follow.ts
+37
-54
data-plane/indexing/plugins/follow.ts
···
26
26
indexedAt: timestamp,
27
27
};
28
28
29
-
// Use findOneAndUpdate with upsert on the compound key to handle potential duplicate key errors
30
-
try {
31
-
const insertedFollow = await db.models.Follow.findOneAndUpdate(
32
-
{
33
-
authorDid: follow.authorDid,
34
-
subject: follow.subject,
35
-
},
36
-
follow,
37
-
{ upsert: true, new: true },
38
-
);
39
-
return insertedFollow;
40
-
} catch (err) {
41
-
const mongoError = err as { code?: number };
42
-
if (mongoError.code === 11000) {
43
-
return null; // Silently skip duplicates
44
-
}
45
-
throw err;
46
-
}
29
+
const insertedFollow = await db.models.Follow.findOneAndUpdate(
30
+
{ uri: follow.uri },
31
+
{ $set: follow },
32
+
{ upsert: true, new: true },
33
+
);
34
+
return insertedFollow;
47
35
};
48
36
49
37
const findDuplicate = async (
···
91
79
};
92
80
93
81
const updateAggregates = async (db: Database, follow: IndexedFollow) => {
94
-
try {
95
-
// Update followers count for the subject (count both types)
96
-
const followersCount = await db.models.Follow.countDocuments({
97
-
subject: follow.subject,
98
-
});
82
+
// Update followers count for the subject (count both types)
83
+
const followersCount = await db.models.Follow.countDocuments({
84
+
subject: follow.subject,
85
+
});
99
86
100
-
// First check if profile exists to avoid creating one with null URI
101
-
const existingSubjectProfile = await db.models.Profile.findOne({
102
-
authorDid: follow.subject,
103
-
});
87
+
// First check if profile exists to avoid creating one with null URI
88
+
const existingSubjectProfile = await db.models.Profile.findOne({
89
+
authorDid: follow.subject,
90
+
});
104
91
105
-
if (existingSubjectProfile) {
106
-
// Only update existing profiles
107
-
await db.models.Profile.findOneAndUpdate(
108
-
{ authorDid: follow.subject },
109
-
{ followersCount },
110
-
{ new: true },
111
-
);
112
-
}
92
+
if (existingSubjectProfile) {
93
+
// Only update existing profiles
94
+
await db.models.Profile.findOneAndUpdate(
95
+
{ authorDid: follow.subject },
96
+
{ $set: { followersCount } },
97
+
{ new: true },
98
+
);
99
+
}
113
100
114
-
// Update follows count for the author (count both types)
115
-
const followsCount = await db.models.Follow.countDocuments({
116
-
authorDid: follow.authorDid,
117
-
});
101
+
// Update follows count for the author (count both types)
102
+
const followsCount = await db.models.Follow.countDocuments({
103
+
authorDid: follow.authorDid,
104
+
});
118
105
119
-
// First check if profile exists to avoid creating one with null URI
120
-
const existingAuthorProfile = await db.models.Profile.findOne({
121
-
authorDid: follow.authorDid,
122
-
});
106
+
// First check if profile exists to avoid creating one with null URI
107
+
const existingAuthorProfile = await db.models.Profile.findOne({
108
+
authorDid: follow.authorDid,
109
+
});
123
110
124
-
if (existingAuthorProfile) {
125
-
// Only update existing profiles
126
-
await db.models.Profile.findOneAndUpdate(
127
-
{ authorDid: follow.authorDid },
128
-
{ followsCount },
129
-
{ new: true },
130
-
);
131
-
}
132
-
} catch (error) {
133
-
console.error("Error updating follow aggregates:", error);
134
-
// Don't throw - allow processing to continue even if aggregates update fails
111
+
if (existingAuthorProfile) {
112
+
// Only update existing profiles
113
+
await db.models.Profile.findOneAndUpdate(
114
+
{ authorDid: follow.authorDid },
115
+
{ $set: { followsCount } },
116
+
{ new: true },
117
+
);
135
118
}
136
119
};
137
120
+6
-16
data-plane/indexing/plugins/generator.ts
+6
-16
data-plane/indexing/plugins/generator.ts
···
40
40
indexedAt: timestamp,
41
41
};
42
42
43
-
// Use findOneAndUpdate with upsert to handle potential duplicate key errors
44
-
try {
45
-
const insertedGenerator = await db.models.Generator.findOneAndUpdate(
46
-
{ uri: generator.uri },
47
-
generator,
48
-
{ upsert: true, new: true },
49
-
);
50
-
return insertedGenerator;
51
-
} catch (err) {
52
-
// Handle duplicate key errors gracefully
53
-
const mongoError = err as { code?: number };
54
-
if (mongoError.code === 11000) {
55
-
return null; // Silently skip duplicates
56
-
}
57
-
throw err;
58
-
}
43
+
const insertedGenerator = await db.models.Generator.findOneAndUpdate(
44
+
{ uri: generator.uri },
45
+
{ $set: generator },
46
+
{ upsert: true, new: true },
47
+
);
48
+
return insertedGenerator;
59
49
};
60
50
61
51
const findDuplicate = (): AtUri | null => {
+41
-57
data-plane/indexing/plugins/like.ts
+41
-57
data-plane/indexing/plugins/like.ts
···
35
35
};
36
36
37
37
// Use findOneAndUpdate with upsert on the compound key to handle potential duplicate key errors
38
-
try {
39
-
const insertedLike = await db.models.Like.findOneAndUpdate(
40
-
{
41
-
authorDid: like.authorDid,
42
-
subject: like.subject,
43
-
},
44
-
like,
45
-
{ upsert: true, new: true },
46
-
);
47
-
return insertedLike;
48
-
} catch (err) {
49
-
// Handle duplicate key errors gracefully
50
-
const mongoError = err as { code?: number };
51
-
if (mongoError.code === 11000) {
52
-
return null; // Silently skip duplicates
53
-
}
54
-
throw err;
55
-
}
38
+
const insertedLike = await db.models.Like.findOneAndUpdate(
39
+
{ uri: like.uri },
40
+
{ $set: like },
41
+
{ upsert: true, new: true },
42
+
);
43
+
return insertedLike;
56
44
};
57
45
58
46
const findDuplicate = async (
···
138
126
};
139
127
140
128
const updateAggregates = async (db: Database, like: IndexedLike) => {
141
-
try {
142
-
const likeCount = await db.models.Like.countDocuments({
143
-
subject: like.subject,
144
-
});
129
+
const likeCount = await db.models.Like.countDocuments({
130
+
subject: like.subject,
131
+
});
145
132
146
-
const subjectUri = new AtUri(like.subject);
133
+
const subjectUri = new AtUri(like.subject);
147
134
148
-
if (subjectUri.collection === "so.sprk.feed.generator") {
149
-
const existingGenerator = await db.models.Generator.findOne({
150
-
uri: like.subject,
151
-
});
135
+
if (subjectUri.collection === "so.sprk.feed.generator") {
136
+
const existingGenerator = await db.models.Generator.findOne({
137
+
uri: like.subject,
138
+
});
152
139
153
-
if (existingGenerator) {
154
-
await db.models.Generator.findOneAndUpdate(
155
-
{ uri: like.subject },
156
-
{ $set: { likeCount } },
157
-
{ new: true },
158
-
);
159
-
}
160
-
} else {
161
-
const existingPost = await db.models.Post.findOne({
162
-
uri: like.subject,
163
-
});
140
+
if (existingGenerator) {
141
+
await db.models.Generator.findOneAndUpdate(
142
+
{ uri: like.subject },
143
+
{ $set: { likeCount } },
144
+
{ new: true },
145
+
);
146
+
}
147
+
} else {
148
+
const existingPost = await db.models.Post.findOne({
149
+
uri: like.subject,
150
+
});
164
151
165
-
if (existingPost) {
166
-
await db.models.Post.findOneAndUpdate(
167
-
{ uri: like.subject },
168
-
{ $set: { likeCount } },
169
-
{ new: true },
170
-
);
171
-
}
152
+
if (existingPost) {
153
+
await db.models.Post.findOneAndUpdate(
154
+
{ uri: like.subject },
155
+
{ $set: { likeCount } },
156
+
{ new: true },
157
+
);
158
+
}
172
159
173
-
const existingReply = await db.models.Reply.findOne({
174
-
uri: like.subject,
175
-
});
160
+
const existingReply = await db.models.Reply.findOne({
161
+
uri: like.subject,
162
+
});
176
163
177
-
if (existingReply) {
178
-
await db.models.Reply.findOneAndUpdate(
179
-
{ uri: like.subject },
180
-
{ $set: { likeCount } },
181
-
{ new: true },
182
-
);
183
-
}
164
+
if (existingReply) {
165
+
await db.models.Reply.findOneAndUpdate(
166
+
{ uri: like.subject },
167
+
{ $set: { likeCount } },
168
+
{ new: true },
169
+
);
184
170
}
185
-
} catch (error) {
186
-
console.error("Error updating like aggregates:", error);
187
171
}
188
172
};
189
173
+75
-102
data-plane/indexing/plugins/post.ts
+75
-102
data-plane/indexing/plugins/post.ts
···
59
59
obj: PostRecord,
60
60
timestamp: string,
61
61
): Promise<IndexedPost | null> => {
62
-
console.log("DEBUG: Post indexing started");
63
-
// Ensure actor record exists before creating post
64
-
const actorExists = await db.models.Actor.findOne({ did: uri.host }).lean();
65
-
if (!actorExists) {
66
-
// This should trigger actor indexing, but for now we'll just log
67
-
console.log(
68
-
`Post indexing: No actor record found for ${uri.host}, post may have missing handle`,
69
-
);
70
-
}
71
-
72
-
console.log("DEBUG: Post media:", JSON.stringify(obj.media, null, 2));
73
-
74
62
const post = {
75
63
uri: uri.toString(),
76
64
cid: cid.toString(),
···
86
74
indexedAt: timestamp,
87
75
};
88
76
89
-
// Use findOneAndUpdate with upsert to handle potential duplicate key errors
90
-
try {
91
-
const insertedPost = await db.models.Post.findOneAndUpdate(
92
-
{ uri: post.uri },
93
-
post,
94
-
{ upsert: true, new: true },
95
-
);
77
+
const insertedPost = await db.models.Post.findOneAndUpdate(
78
+
{ uri: post.uri },
79
+
{ $set: post },
80
+
{ upsert: true, new: true },
81
+
);
96
82
97
-
const facets = (obj.caption?.facets || [])
98
-
.flatMap((facet) => facet.features)
99
-
.flatMap((feature) => {
100
-
if (isMention(feature)) {
101
-
return {
102
-
type: "mention" as const,
103
-
value: feature.did,
104
-
};
105
-
}
106
-
if (isLink(feature)) {
107
-
return {
108
-
type: "link" as const,
109
-
value: feature.uri,
110
-
};
111
-
}
112
-
return [];
113
-
});
114
-
115
-
// Media processing - medias are stored inline in the Post model
116
-
const medias: Array<{
117
-
position?: number;
118
-
imageCid?: string;
119
-
alt?: string | null;
120
-
thumbCid?: string | null;
121
-
videoCid?: string;
122
-
}> = [];
123
-
const postMedias = separateMedia(obj.media);
124
-
for (const postMedia of postMedias) {
125
-
if (isMediaImages(postMedia)) {
126
-
const { images } = postMedia as MediaImages;
127
-
const imagesMedia = images.map((
128
-
img: MediaImage,
129
-
i: number,
130
-
) => ({
131
-
position: i,
132
-
imageCid: img.image.ref.toString(),
133
-
alt: img.alt,
134
-
}));
135
-
medias.push(...imagesMedia);
136
-
} else if (isMediaVideo(postMedia)) {
137
-
const media = postMedia as MediaVideo;
138
-
const videoMedia = {
139
-
postUri: uri.toString(),
140
-
videoCid: media.video.ref.toString(),
141
-
alt: media.alt ?? null,
83
+
const facets = (obj.caption?.facets || [])
84
+
.flatMap((facet) => facet.features)
85
+
.flatMap((feature) => {
86
+
if (isMention(feature)) {
87
+
return {
88
+
type: "mention" as const,
89
+
value: feature.did,
90
+
};
91
+
}
92
+
if (isLink(feature)) {
93
+
return {
94
+
type: "link" as const,
95
+
value: feature.uri,
142
96
};
143
-
medias.push(videoMedia);
144
97
}
145
-
}
146
-
147
-
const descendents = await getDescendents(db, {
148
-
uri: post.uri,
149
-
depth: REPLY_NOTIF_DEPTH,
98
+
return [];
150
99
});
151
100
152
-
return {
153
-
post: insertedPost,
154
-
facets,
155
-
medias,
156
-
descendents,
157
-
};
158
-
} catch (err) {
159
-
// Handle duplicate key errors gracefully
160
-
const mongoError = err as { code?: number };
161
-
if (mongoError.code === 11000) {
162
-
return null; // Silently skip duplicates
101
+
// Media processing - medias are stored inline in the Post model
102
+
const medias: Array<{
103
+
position?: number;
104
+
imageCid?: string;
105
+
alt?: string | null;
106
+
thumbCid?: string | null;
107
+
videoCid?: string;
108
+
}> = [];
109
+
const postMedias = separateMedia(obj.media);
110
+
for (const postMedia of postMedias) {
111
+
if (isMediaImages(postMedia)) {
112
+
const { images } = postMedia as MediaImages;
113
+
const imagesMedia = images.map((
114
+
img: MediaImage,
115
+
i: number,
116
+
) => ({
117
+
position: i,
118
+
imageCid: img.image.ref.toString(),
119
+
alt: img.alt,
120
+
}));
121
+
medias.push(...imagesMedia);
122
+
} else if (isMediaVideo(postMedia)) {
123
+
const media = postMedia as MediaVideo;
124
+
const videoMedia = {
125
+
postUri: uri.toString(),
126
+
videoCid: media.video.ref.toString(),
127
+
alt: media.alt ?? null,
128
+
};
129
+
medias.push(videoMedia);
163
130
}
164
-
throw err;
165
131
}
132
+
133
+
const descendents = await getDescendents(db, {
134
+
uri: post.uri,
135
+
depth: REPLY_NOTIF_DEPTH,
136
+
});
137
+
138
+
return {
139
+
post: insertedPost,
140
+
facets,
141
+
medias,
142
+
descendents,
143
+
};
166
144
};
167
145
168
146
const findDuplicate = (): AtUri | null => {
···
280
258
};
281
259
282
260
const updateAggregates = async (db: Database, postIdx: IndexedPost) => {
283
-
try {
284
-
// Update posts count for author
285
-
const postsCount = await db.models.Post.countDocuments({
286
-
authorDid: postIdx.post.authorDid,
287
-
});
261
+
// Update posts count for author
262
+
const postsCount = await db.models.Post.countDocuments({
263
+
authorDid: postIdx.post.authorDid,
264
+
});
288
265
289
-
// First check if profile exists to avoid creating one with null URI
290
-
const existingProfile = await db.models.Profile.findOne({
291
-
authorDid: postIdx.post.authorDid,
292
-
});
266
+
// First check if profile exists to avoid creating one with null URI
267
+
const existingProfile = await db.models.Profile.findOne({
268
+
authorDid: postIdx.post.authorDid,
269
+
});
293
270
294
-
if (existingProfile) {
295
-
// Only update existing profiles
296
-
await db.models.Profile.findOneAndUpdate(
297
-
{ authorDid: postIdx.post.authorDid },
298
-
{ postsCount },
299
-
{ new: true },
300
-
);
301
-
}
302
-
} catch (error) {
303
-
console.error("Error updating post aggregates:", error);
304
-
// Don't throw - allow processing to continue even if aggregates update fails
271
+
if (existingProfile) {
272
+
// Only update existing profiles
273
+
await db.models.Profile.findOneAndUpdate(
274
+
{ authorDid: postIdx.post.authorDid },
275
+
{ $set: { postsCount } },
276
+
{ new: true },
277
+
);
305
278
}
306
279
};
307
280
+6
-16
data-plane/indexing/plugins/profile.ts
+6
-16
data-plane/indexing/plugins/profile.ts
···
32
32
indexedAt: timestamp,
33
33
};
34
34
35
-
// Use findOneAndUpdate with upsert to handle potential duplicate key errors
36
-
try {
37
-
const insertedProfile = await db.models.Profile.findOneAndUpdate(
38
-
{ uri: profile.uri },
39
-
profile,
40
-
{ upsert: true, new: true },
41
-
);
42
-
return insertedProfile;
43
-
} catch (err) {
44
-
// Handle duplicate key errors gracefully
45
-
const mongoError = err as { code?: number };
46
-
if (mongoError.code === 11000) {
47
-
return null; // Silently skip duplicates
48
-
}
49
-
throw err;
50
-
}
35
+
const insertedProfile = await db.models.Profile.findOneAndUpdate(
36
+
{ uri: profile.uri },
37
+
{ $set: profile },
38
+
{ upsert: true, new: true },
39
+
);
40
+
return insertedProfile;
51
41
};
52
42
53
43
const findDuplicate = (): AtUri | null => {
+61
-80
data-plane/indexing/plugins/reply.ts
+61
-80
data-plane/indexing/plugins/reply.ts
···
56
56
obj: ReplyRecord,
57
57
timestamp: string,
58
58
): Promise<IndexedReply | null> => {
59
-
console.log("DEBUG: Post indexing started");
60
-
// Ensure actor record exists before creating post
61
-
const actorExists = await db.models.Actor.findOne({ did: uri.host }).lean();
62
-
if (!actorExists) {
63
-
// This should trigger actor indexing, but for now we'll just log
64
-
console.log(
65
-
`Post indexing: No actor record found for ${uri.host}, post may have missing handle`,
66
-
);
67
-
}
68
-
69
59
const reply = {
70
60
uri: uri.toString(),
71
61
cid: cid.toString(),
···
93
83
};
94
84
95
85
// Use findOneAndUpdate with upsert to handle potential duplicate key errors
96
-
try {
97
-
const insertedReply = await db.models.Reply.findOneAndUpdate(
98
-
{ uri: reply.uri },
99
-
reply,
100
-
{ upsert: true, new: true },
101
-
);
86
+
const insertedReply = await db.models.Reply.findOneAndUpdate(
87
+
{ uri: reply.uri },
88
+
{ $set: reply },
89
+
{ upsert: true, new: true },
90
+
);
102
91
103
-
if (obj.reply) {
104
-
const { invalidReplyRoot } = await validateReply(
105
-
db,
106
-
obj.reply,
92
+
if (obj.reply) {
93
+
const { invalidReplyRoot } = await validateReply(
94
+
db,
95
+
obj.reply,
96
+
);
97
+
if (invalidReplyRoot) {
98
+
Object.assign(insertedReply, { invalidReplyRoot });
99
+
await db.models.Reply.updateOne(
100
+
{ uri: reply.uri },
101
+
{ $set: { invalidReplyRoot } },
107
102
);
108
-
if (invalidReplyRoot) {
109
-
Object.assign(insertedReply, { invalidReplyRoot });
110
-
await db.models.Reply.updateOne(
111
-
{ uri: reply.uri },
112
-
{ invalidReplyRoot },
113
-
);
114
-
}
115
103
}
104
+
}
116
105
117
-
const facets = (obj.facets || [])
118
-
.flatMap((facet) => facet.features)
119
-
.flatMap((feature) => {
120
-
if (isMention(feature)) {
121
-
return {
122
-
type: "mention" as const,
123
-
value: feature.did,
124
-
};
125
-
}
126
-
if (isLink(feature)) {
127
-
return {
128
-
type: "link" as const,
129
-
value: feature.uri,
130
-
};
131
-
}
132
-
return [];
133
-
});
134
-
135
-
// Embed processing - embeds are stored inline in the Post model
136
-
let media: {
137
-
postUri?: string;
138
-
cid?: string;
139
-
alt?: string;
140
-
} = {};
141
-
if (isMediaImage(obj.media)) {
142
-
const imageMedia = {
143
-
postUri: uri.toString(),
144
-
cid: obj.media.image.ref.toString(),
145
-
alt: obj.media.alt as string,
146
-
};
147
-
media = imageMedia;
148
-
}
149
-
150
-
const ancestors = await getAncestorsAndSelf(db, {
151
-
uri: reply.uri,
152
-
parentHeight: REPLY_NOTIF_DEPTH,
153
-
});
154
-
const descendents = await getDescendents(db, {
155
-
uri: reply.uri,
156
-
depth: REPLY_NOTIF_DEPTH,
106
+
const facets = (obj.facets || [])
107
+
.flatMap((facet) => facet.features)
108
+
.flatMap((feature) => {
109
+
if (isMention(feature)) {
110
+
return {
111
+
type: "mention" as const,
112
+
value: feature.did,
113
+
};
114
+
}
115
+
if (isLink(feature)) {
116
+
return {
117
+
type: "link" as const,
118
+
value: feature.uri,
119
+
};
120
+
}
121
+
return [];
157
122
});
158
123
159
-
return {
160
-
reply: insertedReply,
161
-
facets,
162
-
media,
163
-
ancestors,
164
-
descendents,
124
+
// Embed processing - embeds are stored inline in the Post model
125
+
let media: {
126
+
postUri?: string;
127
+
cid?: string;
128
+
alt?: string;
129
+
} = {};
130
+
if (isMediaImage(obj.media)) {
131
+
const imageMedia = {
132
+
postUri: uri.toString(),
133
+
cid: obj.media.image.ref.toString(),
134
+
alt: obj.media.alt as string,
165
135
};
166
-
} catch (err) {
167
-
// Handle duplicate key errors gracefully
168
-
const mongoError = err as { code?: number };
169
-
if (mongoError.code === 11000) {
170
-
return null; // Silently skip duplicates
171
-
}
172
-
throw err;
136
+
media = imageMedia;
173
137
}
138
+
139
+
const ancestors = await getAncestorsAndSelf(db, {
140
+
uri: reply.uri,
141
+
parentHeight: REPLY_NOTIF_DEPTH,
142
+
});
143
+
const descendents = await getDescendents(db, {
144
+
uri: reply.uri,
145
+
depth: REPLY_NOTIF_DEPTH,
146
+
});
147
+
148
+
return {
149
+
reply: insertedReply,
150
+
facets,
151
+
media,
152
+
ancestors,
153
+
descendents,
154
+
};
174
155
};
175
156
176
157
const findDuplicate = (): AtUri | null => {
+99
-142
data-plane/indexing/plugins/repost.ts
+99
-142
data-plane/indexing/plugins/repost.ts
···
17
17
obj: Repost.Record,
18
18
timestamp: string,
19
19
): Promise<IndexedRepost | null> => {
20
-
try {
21
-
// Handle via property safely with type assertion
22
-
const viaObj = obj.via as { uri: string; cid: string } | undefined;
23
-
const via = viaObj?.uri || null;
24
-
const viaCid = viaObj?.cid || null;
20
+
const viaObj = obj.via as { uri: string; cid: string } | undefined;
21
+
const via = viaObj?.uri || null;
22
+
const viaCid = viaObj?.cid || null;
25
23
26
-
const repost = {
27
-
uri: uri.toString(),
28
-
cid: cid.toString(),
29
-
authorDid: uri.host,
30
-
subject: {
31
-
uri: obj.subject.uri,
32
-
cid: obj.subject.cid,
33
-
},
34
-
via,
35
-
viaCid,
36
-
createdAt: normalizeDatetimeAlways(obj.createdAt),
37
-
indexedAt: timestamp,
38
-
};
24
+
const repost = {
25
+
uri: uri.toString(),
26
+
cid: cid.toString(),
27
+
authorDid: uri.host,
28
+
subject: {
29
+
uri: obj.subject.uri,
30
+
cid: obj.subject.cid,
31
+
},
32
+
via,
33
+
viaCid,
34
+
createdAt: normalizeDatetimeAlways(obj.createdAt),
35
+
indexedAt: timestamp,
36
+
};
39
37
40
-
// Use findOneAndUpdate with compound key to handle potential duplicate key errors
41
-
try {
42
-
const insertedRepost = await db.models.Repost.findOneAndUpdate(
43
-
{
44
-
authorDid: repost.authorDid,
45
-
"subject.uri": repost.subject.uri,
46
-
},
47
-
repost,
48
-
{ upsert: true, new: true },
49
-
);
50
-
return insertedRepost;
51
-
} catch (err) {
52
-
// Handle duplicate key errors gracefully
53
-
const mongoError = err as { code?: number };
54
-
if (mongoError.code === 11000) {
55
-
return null; // Silently skip duplicates
56
-
}
57
-
throw err;
58
-
}
59
-
} catch (error) {
60
-
// Log the error but prevent it from crashing the process
61
-
console.error("Error processing repost:", error);
62
-
return null;
63
-
}
38
+
// Use findOneAndUpdate with compound key to handle potential duplicate key errors
39
+
const insertedRepost = await db.models.Repost.findOneAndUpdate(
40
+
{ uri: repost.uri },
41
+
{ $set: repost },
42
+
{ upsert: true, new: true },
43
+
);
44
+
return insertedRepost;
64
45
};
65
46
66
47
const findDuplicate = async (
···
76
57
};
77
58
78
59
const notifsForInsert = (obj: IndexedRepost) => {
79
-
try {
80
-
const subjectUri = new AtUri(obj.subject.uri);
81
-
// prevent self-notifications
82
-
const isRepostFromSubjectUser = subjectUri.host === obj.authorDid;
83
-
if (isRepostFromSubjectUser) {
84
-
return [];
85
-
}
60
+
const subjectUri = new AtUri(obj.subject.uri);
61
+
// prevent self-notifications
62
+
const isRepostFromSubjectUser = subjectUri.host === obj.authorDid;
63
+
if (isRepostFromSubjectUser) {
64
+
return [];
65
+
}
86
66
87
-
const notifs: Array<{
88
-
did: string;
89
-
reason: string;
90
-
author: string;
91
-
recordUri: string;
92
-
recordCid: string;
93
-
sortAt: string;
94
-
reasonSubject?: string;
95
-
}> = [
96
-
// Notification to the author of the reposted record.
97
-
{
98
-
did: subjectUri.host,
99
-
author: obj.authorDid,
100
-
recordUri: obj.uri,
101
-
recordCid: obj.cid,
102
-
reason: "repost" as const,
103
-
reasonSubject: subjectUri.toString(),
104
-
sortAt: obj.createdAt,
105
-
},
106
-
];
67
+
const notifs: Array<{
68
+
did: string;
69
+
reason: string;
70
+
author: string;
71
+
recordUri: string;
72
+
recordCid: string;
73
+
sortAt: string;
74
+
reasonSubject?: string;
75
+
}> = [
76
+
// Notification to the author of the reposted record.
77
+
{
78
+
did: subjectUri.host,
79
+
author: obj.authorDid,
80
+
recordUri: obj.uri,
81
+
recordCid: obj.cid,
82
+
reason: "repost" as const,
83
+
reasonSubject: subjectUri.toString(),
84
+
sortAt: obj.createdAt,
85
+
},
86
+
];
107
87
108
-
if (obj.via) {
109
-
try {
110
-
const viaUri = new AtUri(obj.via);
111
-
const isRepostFromViaSubjectUser = viaUri.host === obj.authorDid;
112
-
// prevent self-notifications
113
-
if (!isRepostFromViaSubjectUser) {
114
-
notifs.push(
115
-
// Notification to the reposter via whose repost the repost was made.
116
-
{
117
-
did: viaUri.host,
118
-
author: obj.authorDid,
119
-
recordUri: obj.uri,
120
-
recordCid: obj.cid,
121
-
reason: "repost-via-repost" as const,
122
-
reasonSubject: viaUri.toString(),
123
-
sortAt: obj.createdAt,
124
-
},
125
-
);
126
-
}
127
-
} catch (viaError) {
128
-
console.error("Error processing via uri in notification:", viaError);
129
-
// Continue with just the main notification
130
-
}
88
+
if (obj.via) {
89
+
const viaUri = new AtUri(obj.via);
90
+
const isRepostFromViaSubjectUser = viaUri.host === obj.authorDid;
91
+
// prevent self-notifications
92
+
if (!isRepostFromViaSubjectUser) {
93
+
notifs.push(
94
+
// Notification to the reposter via whose repost the repost was made.
95
+
{
96
+
did: viaUri.host,
97
+
author: obj.authorDid,
98
+
recordUri: obj.uri,
99
+
recordCid: obj.cid,
100
+
reason: "repost-via-repost" as const,
101
+
reasonSubject: viaUri.toString(),
102
+
sortAt: obj.createdAt,
103
+
},
104
+
);
131
105
}
106
+
}
132
107
133
-
return notifs;
134
-
} catch (error) {
135
-
console.error("Error generating notifications for insert:", error);
136
-
return [];
137
-
}
108
+
return notifs;
138
109
};
139
110
140
111
const deleteFn = async (
141
112
db: Database,
142
113
uri: AtUri,
143
114
): Promise<IndexedRepost | null> => {
144
-
try {
145
-
const deleted = await db.models.Repost.findOneAndDelete({
146
-
uri: uri.toString(),
147
-
});
148
-
return deleted;
149
-
} catch (error) {
150
-
console.error("Error deleting repost:", error);
151
-
return null;
152
-
}
115
+
const deleted = await db.models.Repost.findOneAndDelete({
116
+
uri: uri.toString(),
117
+
});
118
+
return deleted;
153
119
};
154
120
155
121
const notifsForDelete = (
156
122
deleted: IndexedRepost,
157
123
replacedBy: IndexedRepost | null,
158
124
) => {
159
-
try {
160
-
const toDelete = replacedBy ? [] : [deleted.uri];
161
-
return { notifs: [], toDelete };
162
-
} catch (error) {
163
-
console.error("Error processing notifications for delete:", error);
164
-
return { notifs: [], toDelete: [] };
165
-
}
125
+
const toDelete = replacedBy ? [] : [deleted.uri];
126
+
return { notifs: [], toDelete };
166
127
};
167
128
168
129
const updateAggregates = async (db: Database, repost: IndexedRepost) => {
169
-
try {
170
-
const repostCount = await db.models.Repost.countDocuments({
171
-
"subject.uri": repost.subject.uri,
172
-
});
130
+
const repostCount = await db.models.Repost.countDocuments({
131
+
"subject.uri": repost.subject.uri,
132
+
});
173
133
174
-
const existingPost = await db.models.Post.findOne({
175
-
uri: repost.subject.uri,
176
-
});
134
+
const existingPost = await db.models.Post.findOne({
135
+
uri: repost.subject.uri,
136
+
});
177
137
178
-
if (existingPost) {
179
-
await db.models.Post.findOneAndUpdate(
180
-
{ uri: repost.subject.uri },
181
-
{ $set: { repostCount } },
182
-
{ new: true },
183
-
);
184
-
}
138
+
if (existingPost) {
139
+
await db.models.Post.findOneAndUpdate(
140
+
{ uri: repost.subject.uri },
141
+
{ $set: { repostCount } },
142
+
{ new: true },
143
+
);
144
+
}
185
145
186
-
const authorRepostCount = await db.models.Repost.countDocuments({
187
-
authorDid: repost.authorDid,
188
-
});
146
+
const authorRepostCount = await db.models.Repost.countDocuments({
147
+
authorDid: repost.authorDid,
148
+
});
189
149
190
-
const existingProfile = await db.models.Profile.findOne({
191
-
authorDid: repost.authorDid,
192
-
});
150
+
const existingProfile = await db.models.Profile.findOne({
151
+
authorDid: repost.authorDid,
152
+
});
193
153
194
-
if (existingProfile) {
195
-
await db.models.Profile.findOneAndUpdate(
196
-
{ authorDid: repost.authorDid },
197
-
{ $set: { repostCount: authorRepostCount } },
198
-
{ new: true },
199
-
);
200
-
}
201
-
} catch (error) {
202
-
console.error("Error updating repost aggregates:", error);
154
+
if (existingProfile) {
155
+
await db.models.Profile.findOneAndUpdate(
156
+
{ authorDid: repost.authorDid },
157
+
{ $set: { repostCount: authorRepostCount } },
158
+
{ new: true },
159
+
);
203
160
}
204
161
};
205
162
+8
-21
data-plane/indexing/plugins/story.ts
+8
-21
data-plane/indexing/plugins/story.ts
···
6
6
import { Database } from "../../db/index.ts";
7
7
import { StoryDocument } from "../../db/models.ts";
8
8
import { RecordProcessor } from "../processor.ts";
9
-
import {
10
-
normalizeEmbed,
11
-
normalizeObject,
12
-
} from "../../../utils/embed-normalizer.ts";
13
9
14
10
const lexId = lex.ids.SoSprkStoryPost;
15
11
type IndexedStory = StoryDocument;
···
25
21
uri: uri.toString(),
26
22
cid: cid.toString(),
27
23
authorDid: uri.host,
28
-
media: normalizeEmbed(obj.media) || null,
29
-
sound: normalizeObject(obj.sound) || null,
24
+
media: obj.media,
25
+
sound: obj.sound,
30
26
labels: obj.labels || null,
31
27
tags: obj.tags || [],
32
28
createdAt: normalizeDatetimeAlways(obj.createdAt),
···
34
30
};
35
31
36
32
// Use findOneAndUpdate with upsert to handle potential duplicate key errors
37
-
try {
38
-
const insertedStory = await db.models.Story.findOneAndUpdate(
39
-
{ uri: story.uri },
40
-
story,
41
-
{ upsert: true, new: true },
42
-
);
43
-
return insertedStory;
44
-
} catch (err) {
45
-
// Handle duplicate key errors gracefully
46
-
const mongoError = err as { code?: number };
47
-
if (mongoError.code === 11000) {
48
-
return null; // Silently skip duplicates
49
-
}
50
-
throw err;
51
-
}
33
+
const insertedStory = await db.models.Story.findOneAndUpdate(
34
+
{ uri: story.uri },
35
+
story,
36
+
{ upsert: true, new: true },
37
+
);
38
+
return insertedStory;
52
39
};
53
40
54
41
const findDuplicate = (): AtUri | null => {
+4
-15
data-plane/indexing/processor.ts
+4
-15
data-plane/indexing/processor.ts
···
85
85
}
86
86
}
87
87
88
-
assertValidRecord(obj: unknown, uri: AtUri): asserts obj is T {
89
-
if (!this.matchesCollection(uri)) {
90
-
throw new Error(
91
-
`Record collection mismatch: expected ${this.collection}, got ${uri.collection}`,
92
-
);
93
-
}
94
-
try {
95
-
lexicons.assertValidRecord(this.collection, obj);
96
-
} catch (err) {
97
-
throw new Error(
98
-
`Record validation failed for collection: ${this.collection}. Error: ${err}`,
99
-
);
100
-
}
88
+
assertValidRecord(obj: unknown): asserts obj is T {
89
+
lexicons.assertValidRecord(this.collection, obj);
101
90
}
102
91
103
92
// Helper method to get the lexId this processor handles
···
112
101
timestamp: string,
113
102
opts?: { disableNotifs?: boolean },
114
103
) {
115
-
this.assertValidRecord(obj, uri);
104
+
this.assertValidRecord(obj);
116
105
117
106
// Insert or update record
118
107
await this.db.models.Record.findOneAndUpdate(
···
170
159
timestamp: string,
171
160
opts?: { disableNotifs?: boolean },
172
161
) {
173
-
this.assertValidRecord(obj, uri);
162
+
this.assertValidRecord(obj);
174
163
175
164
// Update record
176
165
await this.db.models.Record.findOneAndUpdate(
+59
-85
data-plane/routes/identity.ts
+59
-85
data-plane/routes/identity.ts
···
45
45
throw new DataPlaneError(Code.InternalError);
46
46
}
47
47
48
-
try {
49
-
const doc = await this.idResolver.did.resolve(did);
50
-
if (!doc) {
51
-
throw new DataPlaneError(Code.NotFound);
52
-
}
53
-
54
-
const result = getResultFromDoc(doc);
55
-
return result;
56
-
} catch (error) {
57
-
console.error("Error resolving DID:", error);
58
-
throw new DataPlaneError(Code.InternalError);
48
+
const doc = await this.idResolver.did.resolve(did);
49
+
if (!doc) {
50
+
throw new DataPlaneError(Code.NotFound);
59
51
}
52
+
53
+
const result = getResultFromDoc(doc);
54
+
return result;
60
55
}
61
56
62
57
async getByHandle(handle: string) {
···
64
59
throw new DataPlaneError(Code.InternalError);
65
60
}
66
61
67
-
try {
68
-
const did = await this.idResolver.handle.resolve(handle);
69
-
if (!did) {
70
-
throw new DataPlaneError(Code.NotFound);
71
-
}
72
-
73
-
const doc = await this.idResolver.did.resolve(did);
74
-
if (!doc || did !== getDid(doc)) {
75
-
throw new DataPlaneError(Code.NotFound);
76
-
}
62
+
const did = await this.idResolver.handle.resolve(handle);
63
+
if (!did) {
64
+
throw new DataPlaneError(Code.NotFound);
65
+
}
77
66
78
-
const result = getResultFromDoc(doc);
79
-
return result;
80
-
} catch (error) {
81
-
console.error("Error resolving handle:", error);
82
-
throw new DataPlaneError(Code.InternalError);
67
+
const doc = await this.idResolver.did.resolve(did);
68
+
if (!doc || did !== getDid(doc)) {
69
+
throw new DataPlaneError(Code.NotFound);
83
70
}
71
+
72
+
const result = getResultFromDoc(doc);
73
+
return result;
84
74
}
85
75
86
76
async resolve(identifier: string, type?: "did" | "handle") {
···
88
78
throw new DataPlaneError(Code.InternalError);
89
79
}
90
80
91
-
try {
92
-
let doc: DidDocument | null = null;
93
-
let resolvedDid: string | null = null;
81
+
let doc: DidDocument | null = null;
82
+
let resolvedDid: string | null = null;
94
83
95
-
// Auto-detect type if not specified
96
-
const identifierType = type ||
97
-
(identifier.startsWith("did:") ? "did" : "handle");
98
-
99
-
if (identifierType === "did") {
100
-
doc = await this.idResolver.did.resolve(identifier);
101
-
resolvedDid = identifier;
102
-
} else {
103
-
resolvedDid = await this.idResolver.handle.resolve(identifier) || null;
104
-
if (resolvedDid) {
105
-
doc = await this.idResolver.did.resolve(resolvedDid);
106
-
}
107
-
}
84
+
// Auto-detect type if not specified
85
+
const identifierType = type ||
86
+
(identifier.startsWith("did:") ? "did" : "handle");
108
87
109
-
if (!doc || (resolvedDid && resolvedDid !== getDid(doc))) {
110
-
throw new DataPlaneError(Code.NotFound);
88
+
if (identifierType === "did") {
89
+
doc = await this.idResolver.did.resolve(identifier);
90
+
resolvedDid = identifier;
91
+
} else {
92
+
resolvedDid = await this.idResolver.handle.resolve(identifier) || null;
93
+
if (resolvedDid) {
94
+
doc = await this.idResolver.did.resolve(resolvedDid);
111
95
}
96
+
}
112
97
113
-
const result = getResultFromDoc(doc);
114
-
return {
115
-
...result,
116
-
resolvedFrom: {
117
-
identifier,
118
-
type: identifierType,
119
-
},
120
-
};
121
-
} catch (error) {
122
-
console.error("Error resolving identity:", error);
123
-
throw new DataPlaneError(Code.InternalError);
98
+
if (!doc || (resolvedDid && resolvedDid !== getDid(doc))) {
99
+
throw new DataPlaneError(Code.NotFound);
124
100
}
101
+
102
+
const result = getResultFromDoc(doc);
103
+
return {
104
+
...result,
105
+
resolvedFrom: {
106
+
identifier,
107
+
type: identifierType,
108
+
},
109
+
};
125
110
}
126
111
127
112
async resolveBatch(
···
133
118
134
119
const results = await Promise.allSettled(
135
120
identifiers.map(async ({ value, type }) => {
136
-
try {
137
-
let doc: DidDocument | null = null;
138
-
let resolvedDid: string | null = null;
139
-
140
-
const identifierType = type ||
141
-
(value.startsWith("did:") ? "did" : "handle");
142
-
143
-
if (identifierType === "did") {
144
-
doc = await this.idResolver!.did.resolve(value);
145
-
resolvedDid = value;
146
-
} else {
147
-
resolvedDid = await this.idResolver!.handle.resolve(value) || null;
148
-
if (resolvedDid) {
149
-
doc = await this.idResolver!.did.resolve(resolvedDid);
150
-
}
151
-
}
121
+
let doc: DidDocument | null = null;
122
+
let resolvedDid: string | undefined;
152
123
153
-
if (!doc || (resolvedDid && resolvedDid !== getDid(doc))) {
154
-
return {
155
-
identifier: value,
156
-
type: identifierType,
157
-
error: "Identity not found",
158
-
};
159
-
}
124
+
const identifierType = type ||
125
+
(value.startsWith("did:") ? "did" : "handle");
126
+
if (identifierType === "did") {
127
+
doc = await this.idResolver!.did.resolve(value);
128
+
resolvedDid = value;
129
+
} else {
130
+
resolvedDid = await this.idResolver!.handle.resolve(value);
131
+
if (!resolvedDid) throw new DataPlaneError(Code.NotFound);
132
+
doc = await this.idResolver!.did.resolve(resolvedDid);
133
+
}
160
134
135
+
if (!doc || (resolvedDid && resolvedDid !== getDid(doc))) {
161
136
return {
162
137
identifier: value,
163
138
type: identifierType,
164
-
...getResultFromDoc(doc),
165
-
};
166
-
} catch (_error) {
167
-
return {
168
-
identifier: value,
169
-
type: type || "unknown",
170
-
error: "Failed to resolve identity",
139
+
error: "Identity not found",
171
140
};
172
141
}
142
+
return {
143
+
identifier: value,
144
+
type: identifierType,
145
+
...getResultFromDoc(doc),
146
+
};
173
147
}),
174
148
);
175
149
+12
-42
data-plane/routes/records.ts
+12
-42
data-plane/routes/records.ts
···
136
136
}
137
137
138
138
async getLikeRecords(uris: string[]) {
139
-
try {
140
-
const result = await getRecords(this.db, uris, ids.SoSprkFeedLike);
141
-
return result;
142
-
} catch (error) {
143
-
console.error("Error fetching like records:", error);
144
-
throw new DataPlaneError(Code.InternalError);
145
-
}
139
+
const result = await getRecords(this.db, uris, ids.SoSprkFeedLike);
140
+
return result;
146
141
}
147
142
148
143
async getPostRecords(uris: string[]) {
149
-
try {
150
-
const result = await getPostRecords(this.db, uris);
151
-
return result;
152
-
} catch (error) {
153
-
console.error("Error fetching post records:", error);
154
-
throw new DataPlaneError(Code.InternalError);
155
-
}
144
+
const result = await getPostRecords(this.db, uris);
145
+
return result;
156
146
}
157
147
158
148
async getReplyRecords(uris: string[]) {
159
-
try {
160
-
const result = await getReplyRecords(this.db, uris);
161
-
return result;
162
-
} catch (error) {
163
-
console.error("Error fetching reply records:", error);
164
-
throw new DataPlaneError(Code.InternalError);
165
-
}
149
+
const result = await getReplyRecords(this.db, uris);
150
+
return result;
166
151
}
167
152
168
153
async getProfileRecords(uris: string[]) {
169
-
try {
170
-
const result = await getRecords(this.db, uris, ids.SoSprkActorProfile);
171
-
return result;
172
-
} catch (error) {
173
-
console.error("Error fetching profile records:", error);
174
-
throw new DataPlaneError(Code.InternalError);
175
-
}
154
+
const result = await getRecords(this.db, uris, ids.SoSprkActorProfile);
155
+
return result;
176
156
}
177
157
178
158
async getRepostRecords(uris: string[]) {
179
-
try {
180
-
const result = await getRecords(this.db, uris, ids.AppBskyFeedRepost);
181
-
return result;
182
-
} catch (error) {
183
-
console.error("Error fetching repost records:", error);
184
-
throw new DataPlaneError(Code.InternalError);
185
-
}
159
+
const result = await getRecords(this.db, uris, ids.AppBskyFeedRepost);
160
+
return result;
186
161
}
187
162
188
163
async getRecords(uris: string[]) {
189
-
try {
190
-
const result = await getRecords(this.db, uris);
191
-
return result;
192
-
} catch (error) {
193
-
console.error("Error fetching records:", error);
194
-
throw new DataPlaneError(Code.InternalError);
195
-
}
164
+
const result = await getRecords(this.db, uris);
165
+
return result;
196
166
}
197
167
}
-1
data-plane/subscription.ts
-1
data-plane/subscription.ts
-5
lex/lexicons.ts
-5
lex/lexicons.ts
···
12868
12868
"description":
12869
12869
"Combinations of post/repost types to include in response.",
12870
12870
"knownValues": [
12871
-
"posts_with_replies",
12872
-
"posts_no_replies",
12873
-
"posts_with_media",
12874
-
"posts_and_author_threads",
12875
12871
"posts_with_video",
12876
12872
],
12877
-
"default": "posts_with_replies",
12878
12873
},
12879
12874
"includePins": {
12880
12875
"type": "boolean",
+1
-5
lex/types/so/sprk/feed/getAuthorFeed.ts
+1
-5
lex/types/so/sprk/feed/getAuthorFeed.ts
···
8
8
limit: number;
9
9
cursor?: string;
10
10
/** Combinations of post/repost types to include in response. */
11
-
filter:
12
-
| "posts_with_replies"
13
-
| "posts_no_replies"
14
-
| "posts_with_media"
15
-
| "posts_and_author_threads"
11
+
filter?:
16
12
| "posts_with_video"
17
13
| (string & globalThis.Record<PropertyKey, never>);
18
14
includePins: boolean;
+1
-6
lexicons/so/sprk/feed/getAuthorFeed.json
+1
-6
lexicons/so/sprk/feed/getAuthorFeed.json
···
21
21
"type": "string",
22
22
"description": "Combinations of post/repost types to include in response.",
23
23
"knownValues": [
24
-
"posts_with_replies",
25
-
"posts_no_replies",
26
-
"posts_with_media",
27
-
"posts_and_author_threads",
28
24
"posts_with_video"
29
-
],
30
-
"default": "posts_with_replies"
25
+
]
31
26
},
32
27
"includePins": {
33
28
"type": "boolean",
-308
utils/embed-normalizer.ts
-308
utils/embed-normalizer.ts
···
1
-
/**
2
-
* Utility functions for normalizing embeds to ensure CID objects are converted to $link format
3
-
* This ensures consistent storage and retrieval of embed data across the application.
4
-
*/
5
-
6
-
interface CidRef {
7
-
$link?: string;
8
-
code?: number;
9
-
version?: number;
10
-
multihash?: Uint8Array;
11
-
bytes?: string;
12
-
toString?: () => string;
13
-
}
14
-
15
-
interface NormalizedCidRef {
16
-
$link: string;
17
-
}
18
-
19
-
interface VideoEmbed {
20
-
$type: "so.sprk.embed.video";
21
-
video?: {
22
-
$type: "blob";
23
-
ref: CidRef;
24
-
mimeType?: string;
25
-
size?: number;
26
-
};
27
-
}
28
-
29
-
interface ImageEmbed {
30
-
$type: "so.sprk.embed.images";
31
-
images?: Array<{
32
-
image: {
33
-
$type: "blob";
34
-
ref: CidRef;
35
-
mimeType?: string;
36
-
size?: number;
37
-
};
38
-
alt?: string;
39
-
aspectRatio?: number;
40
-
}>;
41
-
}
42
-
43
-
interface Profile {
44
-
avatar?: {
45
-
$type?: "blob";
46
-
ref?: NormalizedCidRef | null;
47
-
} | CidRef;
48
-
banner?: {
49
-
$type?: "blob";
50
-
ref?: NormalizedCidRef | null;
51
-
} | CidRef;
52
-
[key: string]: unknown;
53
-
}
54
-
55
-
// Normalize embed to ensure CID objects are converted to $link format
56
-
export function normalizeEmbed(embed: unknown): unknown {
57
-
if (!embed || typeof embed !== "object") return embed;
58
-
59
-
const embedObj = embed as Record<string, unknown>;
60
-
61
-
if (
62
-
embedObj.$type === "so.sprk.media.video" && embedObj.video &&
63
-
typeof embedObj.video === "object"
64
-
) {
65
-
const video = embedObj.video as Record<string, unknown>;
66
-
if (video.ref) {
67
-
const ref = video.ref;
68
-
// If ref is a CID object (has code/version/multihash), convert to $link
69
-
if (
70
-
typeof ref === "object" && ref && !(ref as CidRef).$link &&
71
-
((ref as CidRef).code || (ref as CidRef).version ||
72
-
(ref as CidRef).multihash)
73
-
) {
74
-
const toStringFn = (ref as CidRef).toString;
75
-
76
-
if (toStringFn && typeof toStringFn === "function") {
77
-
const cidString = toStringFn.call(ref);
78
-
// Return cleaned up structure without 'original' field
79
-
return {
80
-
$type: "so.sprk.media.video",
81
-
video: {
82
-
$type: "blob",
83
-
ref: { $link: cidString },
84
-
mimeType: video.mimeType,
85
-
size: video.size,
86
-
},
87
-
};
88
-
} else {
89
-
console.error("DEBUG: Could not convert CID object to string:", ref);
90
-
return embed; // Return original if we can't convert
91
-
}
92
-
} else if ((ref as CidRef).$link) {
93
-
// Already normalized, return cleaned up structure
94
-
return {
95
-
$type: "so.sprk.media.video",
96
-
video: {
97
-
$type: "blob",
98
-
ref: { $link: (ref as CidRef).$link },
99
-
mimeType: video.mimeType,
100
-
size: video.size,
101
-
},
102
-
};
103
-
}
104
-
}
105
-
}
106
-
107
-
if (
108
-
embedObj.$type === "so.sprk.media.images" && Array.isArray(embedObj.images)
109
-
) {
110
-
const normalizedImages = embedObj.images.map((img: unknown) => {
111
-
if (
112
-
typeof img === "object" && img && (img as Record<string, unknown>).image
113
-
) {
114
-
const image = (img as Record<string, unknown>).image as Record<
115
-
string,
116
-
unknown
117
-
>;
118
-
if (image.ref) {
119
-
const ref = image.ref;
120
-
if (
121
-
typeof ref === "object" && ref && !(ref as CidRef).$link &&
122
-
((ref as CidRef).code || (ref as CidRef).version ||
123
-
(ref as CidRef).multihash)
124
-
) {
125
-
const toStringFn = (ref as CidRef).toString;
126
-
127
-
if (toStringFn && typeof toStringFn === "function") {
128
-
const cidString = toStringFn.call(ref);
129
-
return {
130
-
image: {
131
-
$type: "blob",
132
-
ref: { $link: cidString },
133
-
mimeType: image.mimeType,
134
-
size: image.size,
135
-
},
136
-
alt: (img as Record<string, unknown>).alt,
137
-
aspectRatio: (img as Record<string, unknown>).aspectRatio,
138
-
};
139
-
} else {
140
-
console.error(
141
-
"DEBUG: Could not convert CID object to string:",
142
-
ref,
143
-
);
144
-
return img;
145
-
}
146
-
} else if ((ref as CidRef).$link) {
147
-
// Already normalized
148
-
return {
149
-
image: {
150
-
$type: "blob",
151
-
ref: { $link: (ref as CidRef).$link },
152
-
mimeType: image.mimeType,
153
-
size: image.size,
154
-
},
155
-
alt: (img as Record<string, unknown>).alt,
156
-
aspectRatio: (img as Record<string, unknown>).aspectRatio,
157
-
};
158
-
}
159
-
}
160
-
}
161
-
return img;
162
-
});
163
-
164
-
return {
165
-
$type: "so.sprk.media.images",
166
-
images: normalizedImages,
167
-
};
168
-
}
169
-
170
-
return embed;
171
-
}
172
-
173
-
// Normalize a single CID reference to $link format
174
-
export function normalizeCidRef(
175
-
ref: CidRef | null | undefined,
176
-
): NormalizedCidRef | null | undefined {
177
-
if (!ref) return ref;
178
-
179
-
// If it's already in $link format, return as-is
180
-
if (typeof ref === "object" && ref.$link) {
181
-
return ref as NormalizedCidRef;
182
-
}
183
-
184
-
// If it's a CID object (has code/version/multihash), convert to $link
185
-
if (
186
-
typeof ref === "object" && !ref.$link &&
187
-
(ref.code || ref.version || ref.multihash)
188
-
) {
189
-
let cidString: string;
190
-
const toStringFn = ref.toString;
191
-
192
-
if (toStringFn && typeof toStringFn === "function") {
193
-
cidString = toStringFn.call(ref);
194
-
} else {
195
-
console.error("DEBUG: Could not convert CID object to string:", ref);
196
-
return ref as NormalizedCidRef; // Return original if we can't convert
197
-
}
198
-
199
-
return { $link: cidString };
200
-
}
201
-
202
-
// If it's a string, wrap it in $link format
203
-
if (typeof ref === "string") {
204
-
return { $link: ref };
205
-
}
206
-
207
-
return ref as NormalizedCidRef;
208
-
}
209
-
210
-
// Normalize profile data to ensure any CID references are converted to $link format
211
-
export function normalizeProfile(profile: unknown): unknown {
212
-
if (!profile || typeof profile !== "object") return profile;
213
-
214
-
const normalized: Record<string, unknown> = {
215
-
...profile as Record<string, unknown>,
216
-
};
217
-
218
-
// Normalize avatar if present
219
-
if (normalized.avatar) {
220
-
// If avatar is a BlobRef (has ref property), normalize the ref
221
-
if (
222
-
typeof normalized.avatar === "object" && normalized.avatar &&
223
-
"ref" in normalized.avatar
224
-
) {
225
-
const normalizedRef = normalizeCidRef(
226
-
(normalized.avatar as Record<string, unknown>).ref as CidRef,
227
-
);
228
-
if (normalizedRef) {
229
-
// Return only the MediaRef format: { $type: "blob", ref: { $link: string } }
230
-
normalized.avatar = {
231
-
$type: "blob",
232
-
ref: normalizedRef,
233
-
};
234
-
}
235
-
} else {
236
-
// If avatar is just a CID object, wrap it in the proper structure
237
-
const normalizedRef = normalizeCidRef(normalized.avatar as CidRef);
238
-
if (normalizedRef) {
239
-
normalized.avatar = {
240
-
$type: "blob",
241
-
ref: normalizedRef,
242
-
};
243
-
}
244
-
}
245
-
}
246
-
247
-
// Normalize banner if present
248
-
if (normalized.banner) {
249
-
// If banner is a BlobRef (has ref property), normalize the ref
250
-
if (
251
-
typeof normalized.banner === "object" && normalized.banner &&
252
-
"ref" in normalized.banner
253
-
) {
254
-
const normalizedRef = normalizeCidRef(
255
-
(normalized.banner as Record<string, unknown>).ref as CidRef,
256
-
);
257
-
if (normalizedRef) {
258
-
// Return only the MediaRef format: { $type: "blob", ref: { $link: string } }
259
-
normalized.banner = {
260
-
$type: "blob",
261
-
ref: normalizedRef,
262
-
};
263
-
}
264
-
} else {
265
-
// If banner is just a CID object, wrap it in the proper structure
266
-
const normalizedRef = normalizeCidRef(normalized.banner as CidRef);
267
-
if (normalizedRef) {
268
-
normalized.banner = {
269
-
$type: "blob",
270
-
ref: normalizedRef,
271
-
};
272
-
}
273
-
}
274
-
}
275
-
276
-
return normalized;
277
-
}
278
-
279
-
// Normalize any object that might contain CID references
280
-
export function normalizeObject(obj: unknown): unknown {
281
-
if (!obj || typeof obj !== "object") return obj;
282
-
283
-
if (Array.isArray(obj)) {
284
-
return obj.map((item) => normalizeObject(item));
285
-
}
286
-
287
-
const normalized: Record<string, unknown> = {};
288
-
289
-
for (const [key, value] of Object.entries(obj as Record<string, unknown>)) {
290
-
if (
291
-
key === "ref" && typeof value === "object" && value &&
292
-
!(value as CidRef).$link &&
293
-
((value as CidRef).code || (value as CidRef).version ||
294
-
(value as CidRef).multihash)
295
-
) {
296
-
// This looks like a CID object, normalize it
297
-
normalized[key] = normalizeCidRef(value as CidRef);
298
-
} else if (typeof value === "object" && value !== null) {
299
-
// Recursively normalize nested objects
300
-
normalized[key] = normalizeObject(value);
301
-
} else {
302
-
// Keep primitive values as-is
303
-
normalized[key] = value;
304
-
}
305
-
}
306
-
307
-
return normalized;
308
-
}
+22
-3
utils/media-transformer.ts
+22
-3
utils/media-transformer.ts
···
2
2
import {
3
3
ImageMedia,
4
4
PostMedia,
5
+
StoryMedia,
5
6
VideoMappingDocument,
6
7
} from "../data-plane/db/models.ts";
7
8
import { ServerConfig } from "../config.ts";
···
80
81
}
81
82
82
83
export function transformMedia(
83
-
media: PostMedia | undefined,
84
+
media: PostMedia | StoryMedia | undefined,
84
85
authorDid: string,
85
86
cfg: ServerConfig,
86
87
videoMapping?: VideoMappingDocument | null,
···
92
93
}
93
94
94
95
if (media.$type === "so.sprk.media.images") {
95
-
return transformImagesMedia(media, authorDid, options);
96
+
return transformImagesMedia(media as PostMedia, authorDid, options);
97
+
}
98
+
99
+
if (media.$type === "so.sprk.media.image") {
100
+
// Handle single image (used in stories and replies)
101
+
const singleImageMedia = media as StoryMedia;
102
+
if (!singleImageMedia.image) {
103
+
return undefined;
104
+
}
105
+
106
+
return {
107
+
$type: "so.sprk.media.image#view",
108
+
thumb:
109
+
`https://media.sprk.so/img/medium/${authorDid}/${singleImageMedia.image.ref.$link}/webp`,
110
+
fullsize:
111
+
`https://media.sprk.so/img/full/${authorDid}/${singleImageMedia.image.ref.$link}/webp`,
112
+
alt: singleImageMedia.image.alt ?? "",
113
+
aspectRatio: singleImageMedia.image.aspectRatio || undefined,
114
+
} as const;
96
115
}
97
116
98
117
if (media.$type === "so.sprk.media.video") {
99
118
return transformVideoMedia(
100
-
media,
119
+
media as PostMedia,
101
120
authorDid,
102
121
cfg,
103
122
videoMapping,
+1
-14
utils/uris.ts
+1
-14
utils/uris.ts
···
32
32
}
33
33
34
34
export function uriToDid(uri: string) {
35
-
try {
36
-
return new AtUri(uri).hostname;
37
-
} catch (error) {
38
-
console.log(`AtUri parser failed for URI: ${uri}, error:`, error);
39
-
// Handle custom collection namespaces that AtUri might not recognize
40
-
// Extract DID from URI manually as fallback
41
-
const match = uri.match(/^at:\/\/(did:[^\/]+)/);
42
-
if (match) {
43
-
console.log(`Successfully extracted DID using fallback: ${match[1]}`);
44
-
return match[1];
45
-
}
46
-
console.error(`Failed to extract DID from URI: ${uri}`);
47
-
throw new Error(`Invalid AT URI format: ${uri}`);
48
-
}
35
+
return new AtUri(uri).hostname;
49
36
}
50
37
51
38
// @TODO temp fix for proliferation of invalid pinned post values