this repo has no description
feat: add repos and fix bugs
+174
-13
+25
api/src/db/drizzle/0002_tangled_repo.sql
+25
api/src/db/drizzle/0002_tangled_repo.sql
···
1
+
CREATE TABLE "tangled_repo" (
2
+
"did" text NOT NULL,
3
+
"rev" text NOT NULL,
4
+
"rkey" text NOT NULL,
5
+
"cid" text,
6
+
"deleted" boolean NOT NULL,
7
+
"name" text,
8
+
"knot" text,
9
+
"spindle" text,
10
+
"description" text,
11
+
"website" text,
12
+
"topics" text[],
13
+
"source" text,
14
+
"labels" text[],
15
+
"repo_did" text,
16
+
"created_at" timestamp with time zone,
17
+
CONSTRAINT "tangled_repo_did_rkey_pk" PRIMARY KEY("did","rkey"),
18
+
CONSTRAINT "tangled_repo_cid_only_null_if_deleted" CHECK (("tangled_repo"."cid" is not null or "tangled_repo"."deleted" = true)),
19
+
CONSTRAINT "tangled_repo_non_null_fields" CHECK (("tangled_repo"."deleted" = true or ("tangled_repo"."name" is not null and "tangled_repo"."knot" is not null and "tangled_repo"."created_at" is not null)))
20
+
);
21
+
--> statement-breakpoint
22
+
ALTER TABLE "tangled_profile" ALTER COLUMN "cid" DROP NOT NULL;--> statement-breakpoint
23
+
ALTER TABLE "tangled_profile" ALTER COLUMN "bluesky" DROP NOT NULL;--> statement-breakpoint
24
+
ALTER TABLE "tangled_profile" ADD CONSTRAINT "tangled_profile_cid_only_null_if_deleted" CHECK (("tangled_profile"."cid" is not null or "tangled_profile"."deleted" = true));--> statement-breakpoint
25
+
ALTER TABLE "tangled_profile" ADD CONSTRAINT "tangled_profile_non_null_fields" CHECK (("tangled_profile"."deleted" = true or "tangled_profile"."bluesky" is not null));
+1
api/src/db/drizzle/meta/0002_snapshot.json
+1
api/src/db/drizzle/meta/0002_snapshot.json
···
1
+
{"id":"6bf06586-128f-4b67-9e6e-5f3ff761c8d3","prevId":"9f626a39-ec46-47c9-acba-9f815b1066cb","version":"7","dialect":"postgresql","tables":{"public.raw_records":{"name":"raw_records","schema":"","columns":{"id":{"name":"id","type":"uuidv7","primaryKey":true,"notNull":true},"tap_id":{"name":"tap_id","type":"bigint","primaryKey":false,"notNull":true},"received_at":{"name":"received_at","type":"timestamp with time zone","primaryKey":false,"notNull":true,"default":"now()"},"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"collection":{"name":"collection","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"action":{"name":"action","type":"raw_action","typeSchema":"public","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"record":{"name":"record","type":"jsonb","primaryKey":false,"notNull":false}},"indexes":{"raw_records_collection_index":{"name":"raw_records_collection_index","columns":[{"expression":"collection","isExpression":false,"asc":true,"nulls":"last"}],"isUnique":false,"concurrently":false,"method":"btree","with":{}},"raw_records_did_collection_rkey_index":{"name":"raw_records_did_collection_rkey_index","columns":[{"expression":"did","isExpression":false,"asc":true,"nulls":"last"},{"expression":"collection","isExpression":false,"asc":true,"nulls":"last"},{"expression":"rkey","isExpression":false,"asc":true,"nulls":"last"}],"isUnique":false,"concurrently":false,"method":"btree","with":{}}},"foreignKeys":{},"compositePrimaryKeys":{},"uniqueConstraints":{"raw_records_tap_id_unique":{"name":"raw_records_tap_id_unique","nullsNotDistinct":false,"columns":["tap_id"]}},"policies":{},"checkConstraints":{"raw_records_not_null_unless_delete":{"name":"raw_records_not_null_unless_delete","value":"(\"raw_records\".\"action\" = 'delete' or (\"raw_records\".\"cid\" is not null and \"raw_records\".\"record\" is not null))"}},"isRLSEnabled":false},"public.tangled_profile":{"name":"tangled_profile","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":true,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"avatar":{"name":"avatar","type":"text","primaryKey":false,"notNull":false},"description":{"name":"description","type":"text","primaryKey":false,"notNull":false},"links":{"name":"links","type":"text[]","primaryKey":false,"notNull":false},"stats":{"name":"stats","type":"text[]","primaryKey":false,"notNull":false},"bluesky":{"name":"bluesky","type":"boolean","primaryKey":false,"notNull":false},"location":{"name":"location","type":"text","primaryKey":false,"notNull":false},"pinned_repositories":{"name":"pinned_repositories","type":"text[]","primaryKey":false,"notNull":false},"pronouns":{"name":"pronouns","type":"text","primaryKey":false,"notNull":false},"preferred_handle":{"name":"preferred_handle","type":"text","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_profile_cid_only_null_if_deleted":{"name":"tangled_profile_cid_only_null_if_deleted","value":"(\"tangled_profile\".\"cid\" is not null or \"tangled_profile\".\"deleted\" = true)"},"tangled_profile_non_null_fields":{"name":"tangled_profile_non_null_fields","value":"(\"tangled_profile\".\"deleted\" = true or \"tangled_profile\".\"bluesky\" is not null)"}},"isRLSEnabled":false},"public.tangled_repo":{"name":"tangled_repo","schema":"","columns":{"did":{"name":"did","type":"text","primaryKey":false,"notNull":true},"rev":{"name":"rev","type":"text","primaryKey":false,"notNull":true},"rkey":{"name":"rkey","type":"text","primaryKey":false,"notNull":true},"cid":{"name":"cid","type":"text","primaryKey":false,"notNull":false},"deleted":{"name":"deleted","type":"boolean","primaryKey":false,"notNull":true},"name":{"name":"name","type":"text","primaryKey":false,"notNull":false},"knot":{"name":"knot","type":"text","primaryKey":false,"notNull":false},"spindle":{"name":"spindle","type":"text","primaryKey":false,"notNull":false},"description":{"name":"description","type":"text","primaryKey":false,"notNull":false},"website":{"name":"website","type":"text","primaryKey":false,"notNull":false},"topics":{"name":"topics","type":"text[]","primaryKey":false,"notNull":false},"source":{"name":"source","type":"text","primaryKey":false,"notNull":false},"labels":{"name":"labels","type":"text[]","primaryKey":false,"notNull":false},"repo_did":{"name":"repo_did","type":"text","primaryKey":false,"notNull":false},"created_at":{"name":"created_at","type":"timestamp with time zone","primaryKey":false,"notNull":false}},"indexes":{},"foreignKeys":{},"compositePrimaryKeys":{"tangled_repo_did_rkey_pk":{"name":"tangled_repo_did_rkey_pk","columns":["did","rkey"]}},"uniqueConstraints":{},"policies":{},"checkConstraints":{"tangled_repo_cid_only_null_if_deleted":{"name":"tangled_repo_cid_only_null_if_deleted","value":"(\"tangled_repo\".\"cid\" is not null or \"tangled_repo\".\"deleted\" = true)"},"tangled_repo_non_null_fields":{"name":"tangled_repo_non_null_fields","value":"(\"tangled_repo\".\"deleted\" = true or (\"tangled_repo\".\"name\" is not null and \"tangled_repo\".\"knot\" is not null and \"tangled_repo\".\"created_at\" is not null))"}},"isRLSEnabled":false}},"enums":{"public.raw_action":{"name":"raw_action","schema":"public","values":["create","update","delete"]}},"schemas":{},"sequences":{},"roles":{},"policies":{},"views":{},"_meta":{"columns":{},"schemas":{},"tables":{}}}
+7
api/src/db/drizzle/meta/_journal.json
+7
api/src/db/drizzle/meta/_journal.json
+47
-4
api/src/db/tables/tangled.ts
+47
-4
api/src/db/tables/tangled.ts
···
1
-
import { boolean, pgTable, text } from "drizzle-orm/pg-core";
1
+
import { and, eq, isNotNull, or, sql } from "drizzle-orm";
2
+
import { boolean, check, pgTable, primaryKey, text, timestamp } from "drizzle-orm/pg-core";
2
3
3
4
export const tangledProfile = pgTable("tangled_profile", {
4
5
did: text().primaryKey(),
5
6
rev: text().notNull(),
6
7
// no rkey here, because always self
7
-
cid: text().notNull(),
8
+
cid: text(),
8
9
deleted: boolean().notNull(),
9
10
10
11
avatar: text(),
11
12
description: text(),
12
13
links: text().array(),
13
14
stats: text().array(),
14
-
bluesky: boolean().notNull(),
15
+
bluesky: boolean(),
15
16
location: text(),
16
17
pinnedRepositories: text().array(),
17
18
pronouns: text(),
18
19
preferredHandle: text(),
19
-
});
20
+
}, (table) => [
21
+
check("tangled_profile_cid_only_null_if_deleted", or(isNotNull(table.cid), eq(table.deleted, sql`true`))!),
22
+
check("tangled_profile_non_null_fields",
23
+
or(
24
+
eq(table.deleted, sql`true`),
25
+
and(
26
+
isNotNull(table.bluesky)
27
+
)
28
+
)!
29
+
)
30
+
]);
31
+
32
+
export const tangledRepo = pgTable("tangled_repo", {
33
+
did: text().notNull(),
34
+
rev: text().notNull(),
35
+
rkey: text().notNull(),
36
+
cid: text(),
37
+
deleted: boolean().notNull(),
38
+
39
+
name: text(),
40
+
knot: text(),
41
+
spindle: text(),
42
+
description: text(),
43
+
website: text(),
44
+
topics: text().array(),
45
+
source: text(),
46
+
labels: text().array(),
47
+
repoDid: text(),
48
+
createdAt: timestamp({ withTimezone: true }),
49
+
}, (table) => [
50
+
primaryKey({ columns: [table.did, table.rkey] }),
51
+
check("tangled_repo_cid_only_null_if_deleted", or(isNotNull(table.cid), eq(table.deleted, sql`true`))!),
52
+
check("tangled_repo_non_null_fields",
53
+
or(
54
+
eq(table.deleted, sql`true`),
55
+
and(
56
+
isNotNull(table.name),
57
+
isNotNull(table.knot),
58
+
isNotNull(table.createdAt),
59
+
)
60
+
)!
61
+
)
62
+
]);
+20
-8
api/src/ingest/profile.ts
+20
-8
api/src/ingest/profile.ts
···
7
7
8
8
export const ingestProfile: CollectionHandler = async (profiles) => {
9
9
for (const profile of profiles) {
10
+
if (profile.action === "delete") {
11
+
await db.insert(tangledProfile).values({
12
+
did: profile.did,
13
+
rev: profile.rev,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: tangledProfile.did,
18
+
set: {
19
+
rev: profile.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledProfile.rev, profile.rev)
24
+
});
25
+
continue;
26
+
}
27
+
10
28
const validatedRecord = lexiconProfile.$safeParse(profile.record);
11
29
if (!validatedRecord.success) continue;
12
30
const { value: profileRecord } = validatedRecord;
13
-
14
-
if (profile.action === "delete") {
15
-
await db.update(tangledProfile).set({
16
-
deleted: true
17
-
}).where(and(eq(tangledProfile.did, profile.did), lt(tangledProfile.rev, profile.rev)));
18
-
continue;
19
-
}
20
31
21
32
const avatar = profileRecord.avatar ? getBlobCidString(profileRecord.avatar) : null;
22
33
···
51
62
pinnedRepositories: profileRecord.pinnedRepositories,
52
63
pronouns: profileRecord.pronouns,
53
64
preferredHandle: profileRecord.preferredHandle,
54
-
}
65
+
},
66
+
setWhere: lt(tangledProfile.rev, profile.rev)
55
67
})
56
68
}
57
69
}
+71
api/src/ingest/repo.ts
+71
api/src/ingest/repo.ts
···
1
+
import { and, eq, lt } from "drizzle-orm";
2
+
import { getBlobCidString } from "@atproto/lex";
3
+
import type { CollectionHandler } from "./tap";
4
+
import * as lexiconRepo from "../lexicons/sh/tangled/repo.ts";
5
+
import { db } from "../db/index.ts";
6
+
import { tangledRepo } from "../db/tables/tangled.ts";
7
+
8
+
export const ingestRepo: CollectionHandler = async (repos) => {
9
+
for (const repo of repos) {
10
+
if (repo.action === "delete") {
11
+
await db.insert(tangledRepo).values({
12
+
did: repo.did,
13
+
rev: repo.rev,
14
+
rkey: repo.rkey,
15
+
cid: null,
16
+
deleted: true,
17
+
}).onConflictDoUpdate({
18
+
target: [tangledRepo.did, tangledRepo.rkey],
19
+
set: {
20
+
rev: repo.rev,
21
+
cid: null,
22
+
deleted: true
23
+
},
24
+
setWhere: lt(tangledRepo.rev, repo.rev)
25
+
});
26
+
continue;
27
+
}
28
+
29
+
const validatedRecord = lexiconRepo.$safeParse(repo.record);
30
+
if (!validatedRecord.success) continue;
31
+
const { value: repoRecord } = validatedRecord;
32
+
33
+
await db.insert(tangledRepo).values({
34
+
did: repo.did,
35
+
rev: repo.rev,
36
+
rkey: repo.rkey,
37
+
cid: repo.cid,
38
+
deleted: false,
39
+
40
+
name: repoRecord.name,
41
+
knot: repoRecord.knot,
42
+
spindle: repoRecord.spindle,
43
+
description: repoRecord.description,
44
+
website: repoRecord.website,
45
+
topics: repoRecord.topics,
46
+
source: repoRecord.source,
47
+
labels: repoRecord.labels,
48
+
repoDid: repoRecord.repoDid,
49
+
createdAt: new Date(repoRecord.createdAt),
50
+
}).onConflictDoUpdate({
51
+
target: [tangledRepo.did, tangledRepo.rkey],
52
+
set: {
53
+
rev: repo.rev,
54
+
cid: repo.cid,
55
+
deleted: false,
56
+
57
+
name: repoRecord.name,
58
+
knot: repoRecord.knot,
59
+
spindle: repoRecord.spindle,
60
+
description: repoRecord.description,
61
+
website: repoRecord.website,
62
+
topics: repoRecord.topics,
63
+
source: repoRecord.source,
64
+
labels: repoRecord.labels,
65
+
repoDid: repoRecord.repoDid,
66
+
createdAt: new Date(repoRecord.createdAt),
67
+
},
68
+
setWhere: lt(tangledRepo.rev, repo.rev)
69
+
})
70
+
}
71
+
}
+3
-1
api/src/ingest/tap.ts
+3
-1
api/src/ingest/tap.ts
···
4
4
import { rawRecords } from '../db/tables/raw_records.ts';
5
5
import { TAP_URL } from '../lib/constants.ts';
6
6
import { ingestProfile } from './profile.ts';
7
+
import { ingestRepo } from './repo.ts';
7
8
8
9
const tap = new Tap(TAP_URL);
9
10
···
35
36
export type CollectionHandler = (updates: CollectionHandlerArg[]) => Promise<void>;
36
37
37
38
const COLLECTION_HANDLERS: Record<string, CollectionHandler> = {
38
-
"sh.tangled.actor.profile": ingestProfile
39
+
"sh.tangled.actor.profile": ingestProfile,
40
+
"sh.tangled.repo": ingestRepo,
39
41
};
40
42
41
43
async function syncLevelTwoTables(insertedRecords: CollectionHandlerArg[], log = false) {