an attempt to make a lightweight, easily self-hostable, scoped bluesky appview
bsky post index to db
+251
-72
+3
-3
.gitignore
+3
-3
.gitignore
+198
-48
indexserver.ts
+198
-48
indexserver.ts
···
1
1
import { indexHandlerContext } from "./index/types.ts";
2
2
3
-
import { validateRecord } from "./utils/records.ts";
3
+
import { assertRecord, validateRecord } from "./utils/records.ts";
4
4
import { searchParamsToJson, withCors } from "./utils/server.ts";
5
5
import * as IndexServerTypes from "./utils/indexservertypes.ts";
6
6
import { Database } from "jsr:@db/sqlite@0.11";
···
9
9
import { JetstreamManager, SpacedustManager } from "./utils/sharders.ts";
10
10
import { handleSpacedust, SpacedustLinkMessage } from "./index/spacedust.ts";
11
11
import { handleJetstream } from "./index/jetstream.ts";
12
+
import * as ATPAPI from "npm:@atproto/api";
12
13
import { AtUri } from "npm:@atproto/api";
13
14
14
15
export class IndexServerUserManager {
···
44
45
45
46
getDbForDid(did: string): Database | null {
46
47
if (!this.users.has(did)) {
47
-
return null
48
+
return null;
48
49
}
49
50
return this.users.get(did)?.db ?? null;
50
51
}
51
52
52
53
coldStart(db: Database) {
53
-
const rows = db.prepare("SELECT did FROM users").all();
54
-
for (const row of rows) {
55
-
this.addUser(row.did);
54
+
const rows = db.prepare("SELECT did FROM users").all();
55
+
for (const row of rows) {
56
+
this.addUser(row.did);
57
+
}
56
58
}
57
-
}
58
59
}
59
60
60
61
class UserIndexServer {
61
62
did: string;
62
-
db: Database;// | undefined;
63
-
jetstream: JetstreamManager;// | undefined;
64
-
spacedust: SpacedustManager;// | undefined;
63
+
db: Database; // | undefined;
64
+
jetstream: JetstreamManager; // | undefined;
65
+
spacedust: SpacedustManager; // | undefined;
65
66
66
67
constructor(did: string) {
67
68
this.did = did;
···
80
81
indexServerIndexer({
81
82
op,
82
83
doer,
84
+
cid: msg.commit.cid,
83
85
rev,
84
86
aturi,
85
87
value,
···
90
92
this.jetstream.start({
91
93
// for realsies pls get from db or something instead of this shit
92
94
wantedDids: [
93
-
this.did
95
+
this.did,
94
96
// "did:plc:mn45tewwnse5btfftvd3powc",
95
97
// "did:plc:yy6kbriyxtimkjqonqatv2rb",
96
98
// "did:plc:zzhzjga3ab5fcs2vnsv2ist3",
···
114
116
//await connectToJetstream(this.did, this.db);
115
117
this.spacedust = new SpacedustManager((msg: SpacedustLinkMessage) => {
116
118
console.log("Received Spacedust message: ", msg);
119
+
const operation = msg.link.operation;
120
+
117
121
const sourceURI = new AtUri(msg.link.source_record);
118
-
const srcUri = msg.link.source_record
119
-
const srcDid = sourceURI.host
120
-
const srcField = msg.link.source
121
-
const srcCol = sourceURI.collection
122
-
const subjectURI = new AtUri(msg.link.subject)
123
-
const subUri = msg.link.subject
124
-
const subDid = subjectURI.host
125
-
const subCol = subjectURI.collection
122
+
const srcUri = msg.link.source_record;
123
+
const srcDid = sourceURI.host;
124
+
const srcField = msg.link.source;
125
+
const srcCol = sourceURI.collection;
126
+
const subjectURI = new AtUri(msg.link.subject);
127
+
const subUri = msg.link.subject;
128
+
const subDid = subjectURI.host;
129
+
const subCol = subjectURI.collection;
126
130
127
-
this.db.run(
128
-
`INSERT INTO backlink_skeleton (
129
-
srcuri,
130
-
srcdid,
131
-
srcfield,
132
-
srccol,
133
-
suburi,
134
-
subdid,
135
-
subcol
136
-
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
137
-
[
138
-
srcUri, // full AT URI of the source record
139
-
srcDid, // did: of the source
140
-
srcField, // e.g., "reply.parent.uri" or "facets.features.did"
141
-
srcCol, // e.g., "app.bsky.feed.post"
142
-
subUri, // full AT URI of the subject (linked record)
143
-
subDid, // did: of the subject
144
-
subCol, // subject collection (can be inferred or passed)
145
-
]
146
-
);
131
+
if (operation === "delete") {
132
+
this.db.run(
133
+
`DELETE FROM backlink_skeleton
134
+
WHERE srcuri = ? AND srcfield = ? AND suburi = ?`,
135
+
[srcUri, srcField, subUri]
136
+
);
137
+
} else if (operation === "create") {
138
+
this.db.run(
139
+
`INSERT OR REPLACE INTO backlink_skeleton (
140
+
srcuri,
141
+
srcdid,
142
+
srcfield,
143
+
srccol,
144
+
suburi,
145
+
subdid,
146
+
subcol
147
+
) VALUES (?, ?, ?, ?, ?, ?, ?)`,
148
+
[
149
+
srcUri, // full AT URI of the source record
150
+
srcDid, // did: of the source
151
+
srcField, // e.g., "reply.parent.uri" or "facets.features.did"
152
+
srcCol, // e.g., "app.bsky.feed.post"
153
+
subUri, // full AT URI of the subject (linked record)
154
+
subDid, // did: of the subject
155
+
subCol, // subject collection (can be inferred or passed)
156
+
]
157
+
);
158
+
}
147
159
});
148
160
this.spacedust.start({
149
161
wantedSources: [
···
187
199
}
188
200
189
201
// initialize() {
190
-
202
+
191
203
// }
192
204
193
205
// async handleHttpRequest(route: string, req: Request): Promise<Response> {
···
215
227
}
216
228
217
229
function openDbForDid(did: string): Database {
230
+
// TODO: we should disallow non users to open a db
218
231
const path = `./dbs/${did}.sqlite`;
219
232
const db = new Database(path);
220
233
setupUserDb(db);
···
477
490
links: `
478
491
SELECT srcuri, srcdid, srccol
479
492
FROM backlink_skeleton
480
-
WHERE suburi = ? AND subcol = ? AND srcfield = ?
493
+
WHERE suburi = ? AND srccol = ? AND srcfield = ?
481
494
`,
482
495
distinctDids: `
483
496
SELECT DISTINCT srcdid
484
497
FROM backlink_skeleton
485
-
WHERE suburi = ? AND subcol = ? AND srcfield = ?
498
+
WHERE suburi = ? AND srccol = ? AND srcfield = ?
486
499
`,
487
500
count: `
488
501
SELECT COUNT(*) as total
489
502
FROM backlink_skeleton
490
-
WHERE suburi = ? AND subcol = ? AND srcfield = ?
503
+
WHERE suburi = ? AND srccol = ? AND srcfield = ?
491
504
`,
492
505
countDistinctDids: `
493
506
SELECT COUNT(DISTINCT srcdid) as total
494
507
FROM backlink_skeleton
495
-
WHERE suburi = ? AND subcol = ? AND srcfield = ?
508
+
WHERE suburi = ? AND srccol = ? AND srcfield = ?
496
509
`,
497
510
all: `
498
511
SELECT suburi, srccol, COUNT(*) as records, COUNT(DISTINCT srcdid) as distinct_dids
···
500
513
WHERE suburi = ?
501
514
GROUP BY suburi, srccol
502
515
`,
516
+
};
517
+
518
+
export function isDid(str: string): boolean {
519
+
return typeof str === "string" && str.startsWith("did:");
503
520
}
504
521
505
-
export async function constellationAPIHandler(req: Request, did: string): Promise<Response> {
522
+
export async function constellationAPIHandler(req: Request): Promise<Response> {
506
523
const url = new URL(req.url);
507
524
const pathname = url.pathname;
508
525
// const bskyUrl = `https://api.bsky.app${pathname}${url.search}`;
···
510
527
// const constellationMethod = pathname.startsWith("/links")
511
528
// ? pathname.slice("/links".length)
512
529
// : null;
513
-
const searchParams = searchParamsToJson(url.searchParams);
530
+
const searchParams = searchParamsToJson(url.searchParams) as linksQuery;
514
531
const jsonUntyped = searchParams;
532
+
const did = isDid(searchParams.target)
533
+
? searchParams.target
534
+
: new AtUri(searchParams.target).host;
515
535
const db = openDbForDid(did);
516
536
517
537
switch (pathname) {
518
538
case "/links": {
519
539
const jsonTyped = jsonUntyped as linksQuery;
520
540
// probably need to do pagination or something
541
+
console.log(JSON.stringify(jsonTyped, null, 2));
542
+
const field = `${jsonTyped.collection}:${jsonTyped.path.replace(
543
+
/^\./,
544
+
""
545
+
)}`;
521
546
522
-
const rows = db.prepare(SQL.links).all(jsonTyped.target, jsonTyped.collection, jsonTyped.path);
547
+
const rows = db
548
+
.prepare(SQL.links)
549
+
.all(jsonTyped.target, jsonTyped.collection, field);
523
550
524
551
const linking_records: linksRecord[] = rows.map((row: any) => {
525
-
const rkey = row.srcuri.split('/').pop()!;
552
+
const rkey = row.srcuri.split("/").pop()!;
526
553
return {
527
554
did: row.srcdid,
528
555
collection: row.srccol,
···
590
617
}
591
618
}
592
619
620
+
function isImageEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedImages.Main {
621
+
return typeof embed === "object" && embed !== null && "$type" in embed &&
622
+
(embed as any).$type === "app.bsky.embed.images";
623
+
}
624
+
625
+
function isVideoEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedVideo.Main {
626
+
return typeof embed === "object" && embed !== null && "$type" in embed &&
627
+
(embed as any).$type === "app.bsky.embed.video";
628
+
}
629
+
630
+
function isRecordEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedRecord.Main {
631
+
return typeof embed === "object" && embed !== null && "$type" in embed &&
632
+
(embed as any).$type === "app.bsky.embed.record";
633
+
}
634
+
635
+
function isRecordWithMediaEmbed(embed: unknown): embed is ATPAPI.AppBskyEmbedRecordWithMedia.Main {
636
+
return typeof embed === "object" && embed !== null && "$type" in embed &&
637
+
(embed as any).$type === "app.bsky.embed.recordWithMedia";
638
+
}
639
+
640
+
function uncid(anything: any): string | null {
641
+
return (anything as Record<string, unknown>)?.["$link"] as string | null || null;
642
+
}
643
+
644
+
function extractImages(embed: unknown) {
645
+
if (isImageEmbed(embed)) return embed.images;
646
+
if (isRecordWithMediaEmbed(embed) && isImageEmbed(embed.media)) return embed.media.images;
647
+
return [];
648
+
}
649
+
650
+
function extractVideo(embed: unknown) {
651
+
if (isVideoEmbed(embed)) return embed;
652
+
if (isRecordWithMediaEmbed(embed) && isVideoEmbed(embed.media)) return embed.media;
653
+
return null;
654
+
}
655
+
656
+
function extractQuoteUri(embed: unknown): string | null {
657
+
if (isRecordEmbed(embed)) return embed.record.uri;
658
+
if (isRecordWithMediaEmbed(embed)) return embed.record.record.uri;
659
+
return null;
660
+
}
661
+
593
662
export function indexServerIndexer(ctx: indexHandlerContext) {
594
-
const record = validateRecord(ctx.value);
663
+
const record = assertRecord(ctx.value);
664
+
//const record = validateRecord(ctx.value);
665
+
const db = openDbForDid(ctx.doer);
666
+
console.log("indexering")
595
667
switch (record?.$type) {
596
668
case "app.bsky.feed.like": {
669
+
return;
670
+
}
671
+
case "app.bsky.feed.post": {
672
+
console.log("bsky post")
673
+
const stmt = db.prepare(`
674
+
INSERT OR IGNORE INTO app_bsky_feed_post (
675
+
uri, did, cid, rev, createdat, indexedat, json,
676
+
text, replyroot, replyparent, quote,
677
+
imagecount, image1cid, image1mime, image1aspect,
678
+
image2cid, image2mime, image2aspect,
679
+
image3cid, image3mime, image3aspect,
680
+
image4cid, image4mime, image4aspect,
681
+
videocount, videocid, videomime, videoaspect
682
+
) VALUES (?, ?, ?, ?, ?, ?, ?,
683
+
?, ?, ?, ?,
684
+
?, ?, ?, ?,
685
+
?, ?, ?,
686
+
?, ?, ?,
687
+
?, ?, ?,
688
+
?, ?, ?, ?)
689
+
`);
690
+
691
+
const embed = record.embed;
692
+
693
+
const images = extractImages(embed);
694
+
const video = extractVideo(embed);
695
+
const quoteUri = extractQuoteUri(embed);
696
+
try {
697
+
stmt.run(
698
+
ctx.aturi?? null,
699
+
ctx.doer?? null,
700
+
ctx.cid?? null,
701
+
ctx.rev?? null,
702
+
record.createdAt,
703
+
Date.now(),
704
+
JSON.stringify(record),
705
+
706
+
record.text ?? null,
707
+
record.reply?.root?.uri ?? null,
708
+
record.reply?.parent?.uri ?? null,
709
+
710
+
quoteUri,
711
+
712
+
images.length,
713
+
uncid(images[0]?.image?.ref) ?? null,
714
+
images[0]?.image?.mimeType ?? null,
715
+
(images[0]?.aspectRatio && images[0].aspectRatio.width && images[0].aspectRatio.height)
716
+
? `${images[0].aspectRatio.width}:${images[0].aspectRatio.height}`
717
+
: null,
718
+
719
+
uncid(images[1]?.image?.ref) ?? null,
720
+
images[1]?.image?.mimeType ?? null,
721
+
(images[1]?.aspectRatio && images[1].aspectRatio.width && images[1].aspectRatio.height)
722
+
? `${images[1].aspectRatio.width}:${images[1].aspectRatio.height}`
723
+
: null,
724
+
725
+
uncid(images[2]?.image?.ref) ?? null,
726
+
images[2]?.image?.mimeType ?? null,
727
+
(images[2]?.aspectRatio && images[2].aspectRatio.width && images[2].aspectRatio.height)
728
+
? `${images[2].aspectRatio.width}:${images[2].aspectRatio.height}`
729
+
: null,
730
+
731
+
uncid(images[3]?.image?.ref) ?? null,
732
+
images[3]?.image?.mimeType ?? null,
733
+
(images[3]?.aspectRatio && images[3].aspectRatio.width && images[3].aspectRatio.height)
734
+
? `${images[3].aspectRatio.width}:${images[3].aspectRatio.height}`
735
+
: null,
736
+
737
+
uncid(video?.video) ? 1 : 0,
738
+
uncid(video?.video) ?? null,
739
+
uncid(video?.video) ? "video/mp4" : null,
740
+
video?.aspectRatio
741
+
? `${video.aspectRatio.width}:${video.aspectRatio.height}`
742
+
: null
743
+
);
744
+
} catch (err) {
745
+
console.error("stmt.run failed:", err);
746
+
}
597
747
return;
598
748
}
599
749
default: {
+22
-4
main.ts
+22
-4
main.ts
···
29
29
export const systemDB = new Database("system.db");
30
30
setupSystemDb(systemDB);
31
31
32
+
// add me lol
33
+
systemDB.exec(`
34
+
INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus)
35
+
VALUES (
36
+
'did:plc:mn45tewwnse5btfftvd3powc',
37
+
'admin',
38
+
datetime('now'),
39
+
'ready'
40
+
);
41
+
42
+
INSERT OR IGNORE INTO users (did, role, registrationdate, onboardingstatus)
43
+
VALUES (
44
+
'did:web:did12.whey.party',
45
+
'admin',
46
+
datetime('now'),
47
+
'ready'
48
+
);
49
+
`)
50
+
32
51
const userManager = new IndexServerUserManager();
33
52
userManager.coldStart(systemDB)
34
53
···
127
146
// const xrpcMethod = pathname.startsWith("/xrpc/")
128
147
// ? pathname.slice("/xrpc/".length)
129
148
// : null;
130
-
const constellationMethod = pathname.startsWith("/links")
131
-
? pathname.slice("/links".length)
132
-
: null;
149
+
console.log(`request for "${pathname}"`)
150
+
const constellation = pathname.startsWith("/links")
133
151
134
-
if (constellationMethod) {
152
+
if (constellation) {
135
153
return await constellationAPIHandler(req);
136
154
}
137
155
-16
utils/dbsystem.ts
-16
utils/dbsystem.ts
···
33
33
handle TEXT
34
34
);
35
35
${createIndexINE} idx_did_handle ON did(handle);
36
-
37
-
-- A global index for relationships between any two pieces of content
38
-
${createTableINE} backlink_skeleton (
39
-
id INTEGER PRIMARY KEY AUTOINCREMENT,
40
-
srcuri TEXT,
41
-
srcdid TEXT,
42
-
srcfield TEXT,
43
-
srccol TEXT,
44
-
suburi TEXT,
45
-
subdid TEXT,
46
-
subcol TEXT
47
-
);
48
-
${createIndexINE} idx_backlink_subdid_mod ON backlink_skeleton(subdid, srcdid);
49
-
${createIndexINE} idx_backlink_suburi_mod ON backlink_skeleton(suburi, srcdid);
50
-
${createIndexINE} idx_backlink_subdid_filter_mod ON backlink_skeleton(subdid, srccol, srcdid);
51
-
${createIndexINE} idx_backlink_suburi_filter_mod ON backlink_skeleton(suburi, srccol, srcdid);
52
36
`);
53
37
}
+16
utils/dbuser.ts
+16
utils/dbuser.ts
···
131
131
-- User's notification settings declaration
132
132
${createTableINE} app_bsky_notification_declaration ( ${baseColumns}, allowSubscriptions TEXT );
133
133
${createIndexINE} idx_notification_declaration_author ON app_bsky_notification_declaration(did);
134
+
135
+
-- A global index for relationships between any two pieces of content
136
+
${createTableINE} backlink_skeleton (
137
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
138
+
srcuri TEXT,
139
+
srcdid TEXT,
140
+
srcfield TEXT,
141
+
srccol TEXT,
142
+
suburi TEXT,
143
+
subdid TEXT,
144
+
subcol TEXT
145
+
);
146
+
${createIndexINE} idx_backlink_subdid_mod ON backlink_skeleton(subdid, srcdid);
147
+
${createIndexINE} idx_backlink_suburi_mod ON backlink_skeleton(suburi, srcdid);
148
+
${createIndexINE} idx_backlink_subdid_filter_mod ON backlink_skeleton(subdid, srccol, srcdid);
149
+
${createIndexINE} idx_backlink_suburi_filter_mod ON backlink_skeleton(suburi, srccol, srcdid);
134
150
`);
135
151
}
+12
-1
utils/records.ts
+12
-1
utils/records.ts
···
75
75
if (result.success) return result.value;
76
76
return undefined;
77
77
}
78
-
78
+
export function assertRecord<T extends KnownRecordType>(
79
+
record: unknown
80
+
): RecordTypeMap[T] | undefined {
81
+
if (typeof record !== 'object' || record === null) {
82
+
return undefined;
83
+
}
84
+
const type = (record as { $type?: string })?.$type;
85
+
if (typeof type !== 'string' || !(type in recordValidators)) {
86
+
return undefined;
87
+
}
88
+
return record as RecordTypeMap[T];
89
+
}
79
90
export async function resolveRecordFromURI({
80
91
did,
81
92
uri,