this repo has no description
feat: get most of ingest working
+1441
-1
+1
api/package.json
+1
api/package.json
-1
api/src/db/tables/tangled.ts
-1
api/src/db/tables/tangled.ts
···
302
302
color: text(),
303
303
createdAt: timestamp({ withTimezone: true }),
304
304
multiple: boolean(),
305
-
306
305
}, (table) => [
307
306
primaryKey({ columns: [table.did, table.rkey] }),
308
307
check("tangled_label_definition_cid_only_null_if_deleted", or(isNotNull(table.cid), eq(table.deleted, sql`true`))!),
+68
api/src/ingest/artifact.ts
+68
api/src/ingest/artifact.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconArtifact from "../lexicons/sh/tangled/repo/artifact.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledArtifact } from "../db/tables/tangled.ts";
6
+
import { getBlobCidString, getBlobMime, getBlobSize } from "@atproto/lex";
7
+
import { encodeHex } from "@std/encoding";
8
+
9
+
export const ingestArtifact: CollectionHandler = async (artifacts) => {
10
+
for (const artifact of artifacts) {
11
+
if (artifact.action === "delete") {
12
+
await db.insert(tangledArtifact).values({
13
+
did: artifact.did,
14
+
rev: artifact.rev,
15
+
rkey: artifact.rkey,
16
+
cid: null,
17
+
deleted: true,
18
+
}).onConflictDoUpdate({
19
+
target: [tangledArtifact.did, tangledArtifact.rkey],
20
+
set: {
21
+
rev: artifact.rev,
22
+
cid: null,
23
+
deleted: true
24
+
},
25
+
setWhere: lt(tangledArtifact.rev, artifact.rev)
26
+
});
27
+
continue;
28
+
}
29
+
30
+
const validatedRecord = lexiconArtifact.$safeParse(artifact.record);
31
+
if (!validatedRecord.success) continue;
32
+
const { value: artifactRecord } = validatedRecord;
33
+
34
+
await db.insert(tangledArtifact).values({
35
+
did: artifact.did,
36
+
rev: artifact.rev,
37
+
rkey: artifact.rkey,
38
+
cid: artifact.cid,
39
+
deleted: false,
40
+
41
+
name: artifactRecord.name,
42
+
repo: artifactRecord.repo,
43
+
repoDid: artifactRecord.repoDid,
44
+
tag: encodeHex(artifactRecord.tag),
45
+
createdAt: new Date(artifactRecord.createdAt),
46
+
artifactRef: getBlobCidString(artifactRecord.artifact),
47
+
artifactSize: getBlobSize(artifactRecord.artifact),
48
+
artifactMimeType: getBlobMime(artifactRecord.artifact),
49
+
}).onConflictDoUpdate({
50
+
target: [tangledArtifact.did, tangledArtifact.rkey],
51
+
set: {
52
+
rev: artifact.rev,
53
+
cid: artifact.cid,
54
+
deleted: false,
55
+
56
+
name: artifactRecord.name,
57
+
repo: artifactRecord.repo,
58
+
repoDid: artifactRecord.repoDid,
59
+
tag: encodeHex(artifactRecord.tag),
60
+
createdAt: new Date(artifactRecord.createdAt),
61
+
artifactRef: getBlobCidString(artifactRecord.artifact),
62
+
artifactSize: getBlobSize(artifactRecord.artifact),
63
+
artifactMimeType: getBlobMime(artifactRecord.artifact),
64
+
},
65
+
setWhere: lt(tangledArtifact.rev, artifact.rev)
66
+
})
67
+
}
68
+
}
+58
api/src/ingest/collaborator.ts
+58
api/src/ingest/collaborator.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconCollaborator from "../lexicons/sh/tangled/repo/collaborator.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledCollaborator } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestCollaborator: CollectionHandler = async (collaborators) => {
8
+
for (const collaborator of collaborators) {
9
+
if (collaborator.action === "delete") {
10
+
await db.insert(tangledCollaborator).values({
11
+
did: collaborator.did,
12
+
rev: collaborator.rev,
13
+
rkey: collaborator.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledCollaborator.did, tangledCollaborator.rkey],
18
+
set: {
19
+
rev: collaborator.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledCollaborator.rev, collaborator.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconCollaborator.$safeParse(collaborator.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: collaboratorRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledCollaborator).values({
33
+
did: collaborator.did,
34
+
rev: collaborator.rev,
35
+
rkey: collaborator.rkey,
36
+
cid: collaborator.cid,
37
+
deleted: false,
38
+
39
+
subject: collaboratorRecord.subject,
40
+
repo: collaboratorRecord.repo,
41
+
repoDid: collaboratorRecord.repoDid,
42
+
createdAt: new Date(collaboratorRecord.createdAt),
43
+
}).onConflictDoUpdate({
44
+
target: [tangledCollaborator.did, tangledCollaborator.rkey],
45
+
set: {
46
+
rev: collaborator.rev,
47
+
cid: collaborator.cid,
48
+
deleted: false,
49
+
50
+
subject: collaboratorRecord.subject,
51
+
repo: collaboratorRecord.repo,
52
+
repoDid: collaboratorRecord.repoDid,
53
+
createdAt: new Date(collaboratorRecord.createdAt),
54
+
},
55
+
setWhere: lt(tangledCollaborator.rev, collaborator.rev)
56
+
})
57
+
}
58
+
}
+54
api/src/ingest/follow.ts
+54
api/src/ingest/follow.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconFollow from "../lexicons/sh/tangled/graph/follow.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledFollow } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestFollow: CollectionHandler = async (follows) => {
8
+
for (const follow of follows) {
9
+
if (follow.action === "delete") {
10
+
await db.insert(tangledFollow).values({
11
+
did: follow.did,
12
+
rev: follow.rev,
13
+
rkey: follow.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledFollow.did, tangledFollow.rkey],
18
+
set: {
19
+
rev: follow.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledFollow.rev, follow.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconFollow.$safeParse(follow.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: followRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledFollow).values({
33
+
did: follow.did,
34
+
rev: follow.rev,
35
+
rkey: follow.rkey,
36
+
cid: follow.cid,
37
+
deleted: false,
38
+
39
+
subject: followRecord.subject,
40
+
createdAt: new Date(followRecord.createdAt),
41
+
}).onConflictDoUpdate({
42
+
target: [tangledFollow.did, tangledFollow.rkey],
43
+
set: {
44
+
rev: follow.rev,
45
+
cid: follow.cid,
46
+
deleted: false,
47
+
48
+
subject: followRecord.subject,
49
+
createdAt: new Date(followRecord.createdAt),
50
+
},
51
+
setWhere: lt(tangledFollow.rev, follow.rev)
52
+
})
53
+
}
54
+
}
+64
api/src/ingest/issue.ts
+64
api/src/ingest/issue.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconIssue from "../lexicons/sh/tangled/repo/issue.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledIssue } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestIssue: CollectionHandler = async (issues) => {
8
+
for (const issue of issues) {
9
+
if (issue.action === "delete") {
10
+
await db.insert(tangledIssue).values({
11
+
did: issue.did,
12
+
rev: issue.rev,
13
+
rkey: issue.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledIssue.did, tangledIssue.rkey],
18
+
set: {
19
+
rev: issue.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledIssue.rev, issue.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconIssue.$safeParse(issue.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: issueRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledIssue).values({
33
+
did: issue.did,
34
+
rev: issue.rev,
35
+
rkey: issue.rkey,
36
+
cid: issue.cid,
37
+
deleted: false,
38
+
39
+
repo: issueRecord.repo,
40
+
repoDid: issueRecord.repoDid,
41
+
title: issueRecord.title,
42
+
body: issueRecord.body,
43
+
createdAt: new Date(issueRecord.createdAt),
44
+
mentions: issueRecord.mentions,
45
+
references: issueRecord.references,
46
+
}).onConflictDoUpdate({
47
+
target: [tangledIssue.did, tangledIssue.rkey],
48
+
set: {
49
+
rev: issue.rev,
50
+
cid: issue.cid,
51
+
deleted: false,
52
+
53
+
repo: issueRecord.repo,
54
+
repoDid: issueRecord.repoDid,
55
+
title: issueRecord.title,
56
+
body: issueRecord.body,
57
+
createdAt: new Date(issueRecord.createdAt),
58
+
mentions: issueRecord.mentions,
59
+
references: issueRecord.references,
60
+
},
61
+
setWhere: lt(tangledIssue.rev, issue.rev)
62
+
})
63
+
}
64
+
}
+62
api/src/ingest/issueComment.ts
+62
api/src/ingest/issueComment.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconIssueComment from "../lexicons/sh/tangled/repo/issue/comment.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledIssueComment } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestIssueComment: CollectionHandler = async (issueComments) => {
8
+
for (const issueComment of issueComments) {
9
+
if (issueComment.action === "delete") {
10
+
await db.insert(tangledIssueComment).values({
11
+
did: issueComment.did,
12
+
rev: issueComment.rev,
13
+
rkey: issueComment.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledIssueComment.did, tangledIssueComment.rkey],
18
+
set: {
19
+
rev: issueComment.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledIssueComment.rev, issueComment.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconIssueComment.$safeParse(issueComment.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: issueCommentRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledIssueComment).values({
33
+
did: issueComment.did,
34
+
rev: issueComment.rev,
35
+
rkey: issueComment.rkey,
36
+
cid: issueComment.cid,
37
+
deleted: false,
38
+
39
+
issue: issueCommentRecord.issue,
40
+
body: issueCommentRecord.body,
41
+
createdAt: new Date(issueCommentRecord.createdAt),
42
+
replyTo: issueCommentRecord.replyTo,
43
+
mentions: issueCommentRecord.mentions,
44
+
references: issueCommentRecord.references,
45
+
}).onConflictDoUpdate({
46
+
target: [tangledIssueComment.did, tangledIssueComment.rkey],
47
+
set: {
48
+
rev: issueComment.rev,
49
+
cid: issueComment.cid,
50
+
deleted: false,
51
+
52
+
issue: issueCommentRecord.issue,
53
+
body: issueCommentRecord.body,
54
+
createdAt: new Date(issueCommentRecord.createdAt),
55
+
replyTo: issueCommentRecord.replyTo,
56
+
mentions: issueCommentRecord.mentions,
57
+
references: issueCommentRecord.references,
58
+
},
59
+
setWhere: lt(tangledIssueComment.rev, issueComment.rev)
60
+
})
61
+
}
62
+
}
+54
api/src/ingest/issueState.ts
+54
api/src/ingest/issueState.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconIssueState from "../lexicons/sh/tangled/repo/issue/state.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledIssueState } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestIssueState: CollectionHandler = async (issueStates) => {
8
+
for (const issueState of issueStates) {
9
+
if (issueState.action === "delete") {
10
+
await db.insert(tangledIssueState).values({
11
+
did: issueState.did,
12
+
rev: issueState.rev,
13
+
rkey: issueState.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledIssueState.did, tangledIssueState.rkey],
18
+
set: {
19
+
rev: issueState.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledIssueState.rev, issueState.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconIssueState.$safeParse(issueState.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: issueStateRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledIssueState).values({
33
+
did: issueState.did,
34
+
rev: issueState.rev,
35
+
rkey: issueState.rkey,
36
+
cid: issueState.cid,
37
+
deleted: false,
38
+
39
+
issue: issueStateRecord.issue,
40
+
state: issueStateRecord.state,
41
+
}).onConflictDoUpdate({
42
+
target: [tangledIssueState.did, tangledIssueState.rkey],
43
+
set: {
44
+
rev: issueState.rev,
45
+
cid: issueState.cid,
46
+
deleted: false,
47
+
48
+
issue: issueStateRecord.issue,
49
+
state: issueStateRecord.state,
50
+
},
51
+
setWhere: lt(tangledIssueState.rev, issueState.rev)
52
+
})
53
+
}
54
+
}
+52
api/src/ingest/knot.ts
+52
api/src/ingest/knot.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconKnot from "../lexicons/sh/tangled/knot.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledKnot } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestKnot: CollectionHandler = async (knots) => {
8
+
for (const knot of knots) {
9
+
if (knot.action === "delete") {
10
+
await db.insert(tangledKnot).values({
11
+
did: knot.did,
12
+
rev: knot.rev,
13
+
rkey: knot.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledKnot.did, tangledKnot.rkey],
18
+
set: {
19
+
rev: knot.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledKnot.rev, knot.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconKnot.$safeParse(knot.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: knotRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledKnot).values({
33
+
did: knot.did,
34
+
rev: knot.rev,
35
+
rkey: knot.rkey,
36
+
cid: knot.cid,
37
+
deleted: false,
38
+
39
+
createdAt: new Date(knotRecord.createdAt),
40
+
}).onConflictDoUpdate({
41
+
target: [tangledKnot.did, tangledKnot.rkey],
42
+
set: {
43
+
rev: knot.rev,
44
+
cid: knot.cid,
45
+
deleted: false,
46
+
47
+
createdAt: new Date(knotRecord.createdAt),
48
+
},
49
+
setWhere: lt(tangledKnot.rev, knot.rev)
50
+
})
51
+
}
52
+
}
+56
api/src/ingest/knotMember.ts
+56
api/src/ingest/knotMember.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconKnotMember from "../lexicons/sh/tangled/knot/member.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledKnotMember } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestKnotMember: CollectionHandler = async (knotMembers) => {
8
+
for (const knotMember of knotMembers) {
9
+
if (knotMember.action === "delete") {
10
+
await db.insert(tangledKnotMember).values({
11
+
did: knotMember.did,
12
+
rev: knotMember.rev,
13
+
rkey: knotMember.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledKnotMember.did, tangledKnotMember.rkey],
18
+
set: {
19
+
rev: knotMember.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledKnotMember.rev, knotMember.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconKnotMember.$safeParse(knotMember.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: knotMemberRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledKnotMember).values({
33
+
did: knotMember.did,
34
+
rev: knotMember.rev,
35
+
rkey: knotMember.rkey,
36
+
cid: knotMember.cid,
37
+
deleted: false,
38
+
39
+
subject: knotMemberRecord.subject,
40
+
domain: knotMemberRecord.domain,
41
+
createdAt: new Date(knotMemberRecord.createdAt),
42
+
}).onConflictDoUpdate({
43
+
target: [tangledKnotMember.did, tangledKnotMember.rkey],
44
+
set: {
45
+
rev: knotMember.rev,
46
+
cid: knotMember.cid,
47
+
deleted: false,
48
+
49
+
subject: knotMemberRecord.subject,
50
+
domain: knotMemberRecord.domain,
51
+
createdAt: new Date(knotMemberRecord.createdAt),
52
+
},
53
+
setWhere: lt(tangledKnotMember.rev, knotMember.rev)
54
+
})
55
+
}
56
+
}
+66
api/src/ingest/labelDefinition.ts
+66
api/src/ingest/labelDefinition.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconLabelDefinition from "../lexicons/sh/tangled/label/definition.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledLabelDefinition } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestLabelDefinition: CollectionHandler = async (labelDefinitions) => {
8
+
for (const labelDefinition of labelDefinitions) {
9
+
if (labelDefinition.action === "delete") {
10
+
await db.insert(tangledLabelDefinition).values({
11
+
did: labelDefinition.did,
12
+
rev: labelDefinition.rev,
13
+
rkey: labelDefinition.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledLabelDefinition.did, tangledLabelDefinition.rkey],
18
+
set: {
19
+
rev: labelDefinition.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledLabelDefinition.rev, labelDefinition.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconLabelDefinition.$safeParse(labelDefinition.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: labelDefinitionRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledLabelDefinition).values({
33
+
did: labelDefinition.did,
34
+
rev: labelDefinition.rev,
35
+
rkey: labelDefinition.rkey,
36
+
cid: labelDefinition.cid,
37
+
deleted: false,
38
+
39
+
name: labelDefinitionRecord.name,
40
+
valueType: labelDefinitionRecord.valueType.type,
41
+
valueFormat: labelDefinitionRecord.valueType.format,
42
+
valueEnum: labelDefinitionRecord.valueType.enum,
43
+
scope: labelDefinitionRecord.scope,
44
+
color: labelDefinitionRecord.color,
45
+
createdAt: new Date(labelDefinitionRecord.createdAt),
46
+
multiple: labelDefinitionRecord.multiple,
47
+
}).onConflictDoUpdate({
48
+
target: [tangledLabelDefinition.did, tangledLabelDefinition.rkey],
49
+
set: {
50
+
rev: labelDefinition.rev,
51
+
cid: labelDefinition.cid,
52
+
deleted: false,
53
+
54
+
name: labelDefinitionRecord.name,
55
+
valueType: labelDefinitionRecord.valueType.type,
56
+
valueFormat: labelDefinitionRecord.valueType.format,
57
+
valueEnum: labelDefinitionRecord.valueType.enum,
58
+
scope: labelDefinitionRecord.scope,
59
+
color: labelDefinitionRecord.color,
60
+
createdAt: new Date(labelDefinitionRecord.createdAt),
61
+
multiple: labelDefinitionRecord.multiple,
62
+
},
63
+
setWhere: lt(tangledLabelDefinition.rev, labelDefinition.rev)
64
+
})
65
+
}
66
+
}
+58
api/src/ingest/labelOp.ts
+58
api/src/ingest/labelOp.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconLabelOp from "../lexicons/sh/tangled/label/op.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledLabelOp } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestLabelOp: CollectionHandler = async (labelOps) => {
8
+
for (const labelOp of labelOps) {
9
+
if (labelOp.action === "delete") {
10
+
await db.insert(tangledLabelOp).values({
11
+
did: labelOp.did,
12
+
rev: labelOp.rev,
13
+
rkey: labelOp.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledLabelOp.did, tangledLabelOp.rkey],
18
+
set: {
19
+
rev: labelOp.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledLabelOp.rev, labelOp.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconLabelOp.$safeParse(labelOp.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: labelOpRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledLabelOp).values({
33
+
did: labelOp.did,
34
+
rev: labelOp.rev,
35
+
rkey: labelOp.rkey,
36
+
cid: labelOp.cid,
37
+
deleted: false,
38
+
39
+
subject: labelOpRecord.subject,
40
+
performedAt: new Date(labelOpRecord.performedAt),
41
+
add: labelOpRecord.add,
42
+
delete: labelOpRecord.delete,
43
+
}).onConflictDoUpdate({
44
+
target: [tangledLabelOp.did, tangledLabelOp.rkey],
45
+
set: {
46
+
rev: labelOp.rev,
47
+
cid: labelOp.cid,
48
+
deleted: false,
49
+
50
+
subject: labelOpRecord.subject,
51
+
performedAt: new Date(labelOpRecord.performedAt),
52
+
add: labelOpRecord.add,
53
+
delete: labelOpRecord.delete,
54
+
},
55
+
setWhere: lt(tangledLabelOp.rev, labelOp.rev)
56
+
})
57
+
}
58
+
}
+82
api/src/ingest/pipeline.ts
+82
api/src/ingest/pipeline.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconPipeline from "../lexicons/sh/tangled/pipeline.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledPipeline } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestPipeline: CollectionHandler = async (pipelines) => {
8
+
for (const pipeline of pipelines) {
9
+
if (pipeline.action === "delete") {
10
+
await db.insert(tangledPipeline).values({
11
+
did: pipeline.did,
12
+
rev: pipeline.rev,
13
+
rkey: pipeline.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledPipeline.did, tangledPipeline.rkey],
18
+
set: {
19
+
rev: pipeline.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledPipeline.rev, pipeline.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconPipeline.$safeParse(pipeline.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: pipelineRecord } = validatedRecord;
31
+
32
+
const manualInputs = pipelineRecord.triggerMetadata.manual?.inputs;
33
+
34
+
await db.insert(tangledPipeline).values({
35
+
did: pipeline.did,
36
+
rev: pipeline.rev,
37
+
rkey: pipeline.rkey,
38
+
cid: pipeline.cid,
39
+
deleted: false,
40
+
41
+
triggerMetadataKind: pipelineRecord.triggerMetadata.kind,
42
+
triggerMetadataRepoKnot: pipelineRecord.triggerMetadata.repo.knot,
43
+
triggerMetadataRepoDid: pipelineRecord.triggerMetadata.repo.did,
44
+
triggerMetadataRepoRepoDid: pipelineRecord.triggerMetadata.repo.repoDid,
45
+
triggerMetadataRepoRepo: pipelineRecord.triggerMetadata.repo.repo,
46
+
triggerMetadataRepoDefaultBranch: pipelineRecord.triggerMetadata.repo.defaultBranch,
47
+
triggerMetadataPushRef: pipelineRecord.triggerMetadata.push?.ref,
48
+
triggerMetadataPushNewSha: pipelineRecord.triggerMetadata.push?.newSha,
49
+
triggerMetadataPushOldSha: pipelineRecord.triggerMetadata.push?.oldSha,
50
+
triggerMetadataPullSourceBranch: pipelineRecord.triggerMetadata.pullRequest?.sourceBranch,
51
+
triggerMetadataPullTargetBranch: pipelineRecord.triggerMetadata.pullRequest?.targetBranch,
52
+
triggerMetadataPullSourceSha: pipelineRecord.triggerMetadata.pullRequest?.sourceSha,
53
+
triggerMetadataPullAction: pipelineRecord.triggerMetadata.pullRequest?.action,
54
+
triggerMetadataManual: manualInputs ? manualInputs.map(({ key, value }) => ({ key, value })) : null,
55
+
workflows: pipelineRecord.workflows,
56
+
}).onConflictDoUpdate({
57
+
target: [tangledPipeline.did, tangledPipeline.rkey],
58
+
set: {
59
+
rev: pipeline.rev,
60
+
cid: pipeline.cid,
61
+
deleted: false,
62
+
63
+
triggerMetadataKind: pipelineRecord.triggerMetadata.kind,
64
+
triggerMetadataRepoKnot: pipelineRecord.triggerMetadata.repo.knot,
65
+
triggerMetadataRepoDid: pipelineRecord.triggerMetadata.repo.did,
66
+
triggerMetadataRepoRepoDid: pipelineRecord.triggerMetadata.repo.repoDid,
67
+
triggerMetadataRepoRepo: pipelineRecord.triggerMetadata.repo.repo,
68
+
triggerMetadataRepoDefaultBranch: pipelineRecord.triggerMetadata.repo.defaultBranch,
69
+
triggerMetadataPushRef: pipelineRecord.triggerMetadata.push?.ref,
70
+
triggerMetadataPushNewSha: pipelineRecord.triggerMetadata.push?.newSha,
71
+
triggerMetadataPushOldSha: pipelineRecord.triggerMetadata.push?.oldSha,
72
+
triggerMetadataPullSourceBranch: pipelineRecord.triggerMetadata.pullRequest?.sourceBranch,
73
+
triggerMetadataPullTargetBranch: pipelineRecord.triggerMetadata.pullRequest?.targetBranch,
74
+
triggerMetadataPullSourceSha: pipelineRecord.triggerMetadata.pullRequest?.sourceSha,
75
+
triggerMetadataPullAction: pipelineRecord.triggerMetadata.pullRequest?.action,
76
+
triggerMetadataManual: manualInputs ? manualInputs.map(({ key, value }) => ({ key, value })) : null,
77
+
workflows: pipelineRecord.workflows,
78
+
},
79
+
setWhere: lt(tangledPipeline.rev, pipeline.rev)
80
+
})
81
+
}
82
+
}
+62
api/src/ingest/pipelineStatus.ts
+62
api/src/ingest/pipelineStatus.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconPipelineStatus from "../lexicons/sh/tangled/pipeline/status.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledPipelineStatus } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestPipelineStatus: CollectionHandler = async (pipelineStatuses) => {
8
+
for (const pipelineStatus of pipelineStatuses) {
9
+
if (pipelineStatus.action === "delete") {
10
+
await db.insert(tangledPipelineStatus).values({
11
+
did: pipelineStatus.did,
12
+
rev: pipelineStatus.rev,
13
+
rkey: pipelineStatus.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledPipelineStatus.did, tangledPipelineStatus.rkey],
18
+
set: {
19
+
rev: pipelineStatus.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledPipelineStatus.rev, pipelineStatus.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconPipelineStatus.$safeParse(pipelineStatus.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: pipelineStatusRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledPipelineStatus).values({
33
+
did: pipelineStatus.did,
34
+
rev: pipelineStatus.rev,
35
+
rkey: pipelineStatus.rkey,
36
+
cid: pipelineStatus.cid,
37
+
deleted: false,
38
+
39
+
pipeline: pipelineStatusRecord.pipeline,
40
+
workflow: pipelineStatusRecord.workflow,
41
+
status: pipelineStatusRecord.status,
42
+
createdAt: new Date(pipelineStatusRecord.createdAt),
43
+
error: pipelineStatusRecord.error,
44
+
exitCode: pipelineStatusRecord.exitCode,
45
+
}).onConflictDoUpdate({
46
+
target: [tangledPipelineStatus.did, tangledPipelineStatus.rkey],
47
+
set: {
48
+
rev: pipelineStatus.rev,
49
+
cid: pipelineStatus.cid,
50
+
deleted: false,
51
+
52
+
pipeline: pipelineStatusRecord.pipeline,
53
+
workflow: pipelineStatusRecord.workflow,
54
+
status: pipelineStatusRecord.status,
55
+
createdAt: new Date(pipelineStatusRecord.createdAt),
56
+
error: pipelineStatusRecord.error,
57
+
exitCode: pipelineStatusRecord.exitCode,
58
+
},
59
+
setWhere: lt(tangledPipelineStatus.rev, pipelineStatus.rev)
60
+
})
61
+
}
62
+
}
+56
api/src/ingest/publicKey.ts
+56
api/src/ingest/publicKey.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconPublicKey from "../lexicons/sh/tangled/publicKey.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledPublicKey } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestPublicKey: CollectionHandler = async (publicKeys) => {
8
+
for (const publicKey of publicKeys) {
9
+
if (publicKey.action === "delete") {
10
+
await db.insert(tangledPublicKey).values({
11
+
did: publicKey.did,
12
+
rev: publicKey.rev,
13
+
rkey: publicKey.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledPublicKey.did, tangledPublicKey.rkey],
18
+
set: {
19
+
rev: publicKey.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledPublicKey.rev, publicKey.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconPublicKey.$safeParse(publicKey.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: publicKeyRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledPublicKey).values({
33
+
did: publicKey.did,
34
+
rev: publicKey.rev,
35
+
rkey: publicKey.rkey,
36
+
cid: publicKey.cid,
37
+
deleted: false,
38
+
39
+
key: publicKeyRecord.key,
40
+
name: publicKeyRecord.name,
41
+
createdAt: new Date(publicKeyRecord.createdAt),
42
+
}).onConflictDoUpdate({
43
+
target: [tangledPublicKey.did, tangledPublicKey.rkey],
44
+
set: {
45
+
rev: publicKey.rev,
46
+
cid: publicKey.cid,
47
+
deleted: false,
48
+
49
+
key: publicKeyRecord.key,
50
+
name: publicKeyRecord.name,
51
+
createdAt: new Date(publicKeyRecord.createdAt),
52
+
},
53
+
setWhere: lt(tangledPublicKey.rev, publicKey.rev)
54
+
})
55
+
}
56
+
}
+76
api/src/ingest/pull.ts
+76
api/src/ingest/pull.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconPull from "../lexicons/sh/tangled/repo/pull.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledPull } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestPull: CollectionHandler = async (pulls) => {
8
+
for (const pull of pulls) {
9
+
if (pull.action === "delete") {
10
+
await db.insert(tangledPull).values({
11
+
did: pull.did,
12
+
rev: pull.rev,
13
+
rkey: pull.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledPull.did, tangledPull.rkey],
18
+
set: {
19
+
rev: pull.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledPull.rev, pull.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconPull.$safeParse(pull.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: pullRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledPull).values({
33
+
did: pull.did,
34
+
rev: pull.rev,
35
+
rkey: pull.rkey,
36
+
cid: pull.cid,
37
+
deleted: false,
38
+
39
+
title: pullRecord.title,
40
+
body: pullRecord.body,
41
+
rounds: pullRecord.rounds,
42
+
sourceBranch: pullRecord.source?.branch,
43
+
sourceRepo: pullRecord.source?.repo,
44
+
sourceRepoDid: pullRecord.source?.repoDid,
45
+
targetBranch: pullRecord.target?.branch,
46
+
targetRepo: pullRecord.target?.repo,
47
+
targetRepoDid: pullRecord.target?.repoDid,
48
+
createdAt: new Date(pullRecord.createdAt),
49
+
mentions: pullRecord.mentions,
50
+
references: pullRecord.references,
51
+
dependentOn: pullRecord.dependentOn,
52
+
}).onConflictDoUpdate({
53
+
target: [tangledPull.did, tangledPull.rkey],
54
+
set: {
55
+
rev: pull.rev,
56
+
cid: pull.cid,
57
+
deleted: false,
58
+
59
+
title: pullRecord.title,
60
+
body: pullRecord.body,
61
+
rounds: pullRecord.rounds,
62
+
sourceBranch: pullRecord.source?.branch,
63
+
sourceRepo: pullRecord.source?.repo,
64
+
sourceRepoDid: pullRecord.source?.repoDid,
65
+
targetBranch: pullRecord.target?.branch,
66
+
targetRepo: pullRecord.target?.repo,
67
+
targetRepoDid: pullRecord.target?.repoDid,
68
+
createdAt: new Date(pullRecord.createdAt),
69
+
mentions: pullRecord.mentions,
70
+
references: pullRecord.references,
71
+
dependentOn: pullRecord.dependentOn,
72
+
},
73
+
setWhere: lt(tangledPull.rev, pull.rev)
74
+
})
75
+
}
76
+
}
+60
api/src/ingest/pullComment.ts
+60
api/src/ingest/pullComment.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconPullComment from "../lexicons/sh/tangled/repo/pull/comment.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledPullComment } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestPullComment: CollectionHandler = async (pullComments) => {
8
+
for (const pullComment of pullComments) {
9
+
if (pullComment.action === "delete") {
10
+
await db.insert(tangledPullComment).values({
11
+
did: pullComment.did,
12
+
rev: pullComment.rev,
13
+
rkey: pullComment.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledPullComment.did, tangledPullComment.rkey],
18
+
set: {
19
+
rev: pullComment.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledPullComment.rev, pullComment.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconPullComment.$safeParse(pullComment.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: pullCommentRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledPullComment).values({
33
+
did: pullComment.did,
34
+
rev: pullComment.rev,
35
+
rkey: pullComment.rkey,
36
+
cid: pullComment.cid,
37
+
deleted: false,
38
+
39
+
pull: pullCommentRecord.pull,
40
+
body: pullCommentRecord.body,
41
+
createdAt: new Date(pullCommentRecord.createdAt),
42
+
mentions: pullCommentRecord.mentions,
43
+
references: pullCommentRecord.references,
44
+
}).onConflictDoUpdate({
45
+
target: [tangledPullComment.did, tangledPullComment.rkey],
46
+
set: {
47
+
rev: pullComment.rev,
48
+
cid: pullComment.cid,
49
+
deleted: false,
50
+
51
+
pull: pullCommentRecord.pull,
52
+
body: pullCommentRecord.body,
53
+
createdAt: new Date(pullCommentRecord.createdAt),
54
+
mentions: pullCommentRecord.mentions,
55
+
references: pullCommentRecord.references,
56
+
},
57
+
setWhere: lt(tangledPullComment.rev, pullComment.rev)
58
+
})
59
+
}
60
+
}
+54
api/src/ingest/pullStatus.ts
+54
api/src/ingest/pullStatus.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconPullStatus from "../lexicons/sh/tangled/repo/pull/status.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledPullStatus } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestPullStatus: CollectionHandler = async (pullStatuses) => {
8
+
for (const pullStatus of pullStatuses) {
9
+
if (pullStatus.action === "delete") {
10
+
await db.insert(tangledPullStatus).values({
11
+
did: pullStatus.did,
12
+
rev: pullStatus.rev,
13
+
rkey: pullStatus.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledPullStatus.did, tangledPullStatus.rkey],
18
+
set: {
19
+
rev: pullStatus.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledPullStatus.rev, pullStatus.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconPullStatus.$safeParse(pullStatus.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: pullStatusRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledPullStatus).values({
33
+
did: pullStatus.did,
34
+
rev: pullStatus.rev,
35
+
rkey: pullStatus.rkey,
36
+
cid: pullStatus.cid,
37
+
deleted: false,
38
+
39
+
pull: pullStatusRecord.pull,
40
+
status: pullStatusRecord.status,
41
+
}).onConflictDoUpdate({
42
+
target: [tangledPullStatus.did, tangledPullStatus.rkey],
43
+
set: {
44
+
rev: pullStatus.rev,
45
+
cid: pullStatus.cid,
46
+
deleted: false,
47
+
48
+
pull: pullStatusRecord.pull,
49
+
status: pullStatusRecord.status,
50
+
},
51
+
setWhere: lt(tangledPullStatus.rev, pullStatus.rev)
52
+
})
53
+
}
54
+
}
+56
api/src/ingest/reaction.ts
+56
api/src/ingest/reaction.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconReaction from "../lexicons/sh/tangled/feed/reaction.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledReaction } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestReaction: CollectionHandler = async (reactions) => {
8
+
for (const reaction of reactions) {
9
+
if (reaction.action === "delete") {
10
+
await db.insert(tangledReaction).values({
11
+
did: reaction.did,
12
+
rev: reaction.rev,
13
+
rkey: reaction.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledReaction.did, tangledReaction.rkey],
18
+
set: {
19
+
rev: reaction.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledReaction.rev, reaction.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconReaction.$safeParse(reaction.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: reactionRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledReaction).values({
33
+
did: reaction.did,
34
+
rev: reaction.rev,
35
+
rkey: reaction.rkey,
36
+
cid: reaction.cid,
37
+
deleted: false,
38
+
39
+
subject: reactionRecord.subject,
40
+
reaction: reactionRecord.reaction,
41
+
createdAt: new Date(reactionRecord.createdAt),
42
+
}).onConflictDoUpdate({
43
+
target: [tangledReaction.did, tangledReaction.rkey],
44
+
set: {
45
+
rev: reaction.rev,
46
+
cid: reaction.cid,
47
+
deleted: false,
48
+
49
+
subject: reactionRecord.subject,
50
+
reaction: reactionRecord.reaction,
51
+
createdAt: new Date(reactionRecord.createdAt),
52
+
},
53
+
setWhere: lt(tangledReaction.rev, reaction.rev)
54
+
})
55
+
}
56
+
}
+70
api/src/ingest/refUpdate.ts
+70
api/src/ingest/refUpdate.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconRefUpdate from "../lexicons/sh/tangled/git/refUpdate.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledRefUpdate } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestRefUpdate: CollectionHandler = async (refUpdates) => {
8
+
for (const refUpdate of refUpdates) {
9
+
if (refUpdate.action === "delete") {
10
+
await db.insert(tangledRefUpdate).values({
11
+
did: refUpdate.did,
12
+
rev: refUpdate.rev,
13
+
rkey: refUpdate.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledRefUpdate.did, tangledRefUpdate.rkey],
18
+
set: {
19
+
rev: refUpdate.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledRefUpdate.rev, refUpdate.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconRefUpdate.$safeParse(refUpdate.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: refUpdateRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledRefUpdate).values({
33
+
did: refUpdate.did,
34
+
rev: refUpdate.rev,
35
+
rkey: refUpdate.rkey,
36
+
cid: refUpdate.cid,
37
+
deleted: false,
38
+
39
+
ref: refUpdateRecord.ref,
40
+
committerDid: refUpdateRecord.committerDid,
41
+
ownerDid: refUpdateRecord.ownerDid,
42
+
repoDid: refUpdateRecord.repoDid,
43
+
repoName: refUpdateRecord.repoName,
44
+
oldSha: refUpdateRecord.oldSha,
45
+
newSha: refUpdateRecord.newSha,
46
+
metaIsDefaultRef: refUpdateRecord.meta.isDefaultRef,
47
+
metaLangBreakdown: refUpdateRecord.meta.langBreakdown,
48
+
metaCommitCount: refUpdateRecord.meta.commitCount,
49
+
}).onConflictDoUpdate({
50
+
target: [tangledRefUpdate.did, tangledRefUpdate.rkey],
51
+
set: {
52
+
rev: refUpdate.rev,
53
+
cid: refUpdate.cid,
54
+
deleted: false,
55
+
56
+
ref: refUpdateRecord.ref,
57
+
committerDid: refUpdateRecord.committerDid,
58
+
ownerDid: refUpdateRecord.ownerDid,
59
+
repoDid: refUpdateRecord.repoDid,
60
+
repoName: refUpdateRecord.repoName,
61
+
oldSha: refUpdateRecord.oldSha,
62
+
newSha: refUpdateRecord.newSha,
63
+
metaIsDefaultRef: refUpdateRecord.meta.isDefaultRef,
64
+
metaLangBreakdown: refUpdateRecord.meta.langBreakdown,
65
+
metaCommitCount: refUpdateRecord.meta.commitCount,
66
+
},
67
+
setWhere: lt(tangledRefUpdate.rev, refUpdate.rev)
68
+
})
69
+
}
70
+
}
+52
api/src/ingest/spindle.ts
+52
api/src/ingest/spindle.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconSpindle from "../lexicons/sh/tangled/spindle.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledSpindle } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestSpindle: CollectionHandler = async (spindles) => {
8
+
for (const spindle of spindles) {
9
+
if (spindle.action === "delete") {
10
+
await db.insert(tangledSpindle).values({
11
+
did: spindle.did,
12
+
rev: spindle.rev,
13
+
rkey: spindle.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledSpindle.did, tangledSpindle.rkey],
18
+
set: {
19
+
rev: spindle.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledSpindle.rev, spindle.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconSpindle.$safeParse(spindle.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: spindleRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledSpindle).values({
33
+
did: spindle.did,
34
+
rev: spindle.rev,
35
+
rkey: spindle.rkey,
36
+
cid: spindle.cid,
37
+
deleted: false,
38
+
39
+
createdAt: new Date(spindleRecord.createdAt),
40
+
}).onConflictDoUpdate({
41
+
target: [tangledSpindle.did, tangledSpindle.rkey],
42
+
set: {
43
+
rev: spindle.rev,
44
+
cid: spindle.cid,
45
+
deleted: false,
46
+
47
+
createdAt: new Date(spindleRecord.createdAt),
48
+
},
49
+
setWhere: lt(tangledSpindle.rev, spindle.rev)
50
+
})
51
+
}
52
+
}
+56
api/src/ingest/spindleMember.ts
+56
api/src/ingest/spindleMember.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconSpindleMember from "../lexicons/sh/tangled/spindle/member.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledSpindleMember } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestSpindleMember: CollectionHandler = async (spindleMembers) => {
8
+
for (const spindleMember of spindleMembers) {
9
+
if (spindleMember.action === "delete") {
10
+
await db.insert(tangledSpindleMember).values({
11
+
did: spindleMember.did,
12
+
rev: spindleMember.rev,
13
+
rkey: spindleMember.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledSpindleMember.did, tangledSpindleMember.rkey],
18
+
set: {
19
+
rev: spindleMember.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledSpindleMember.rev, spindleMember.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconSpindleMember.$safeParse(spindleMember.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: spindleMemberRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledSpindleMember).values({
33
+
did: spindleMember.did,
34
+
rev: spindleMember.rev,
35
+
rkey: spindleMember.rkey,
36
+
cid: spindleMember.cid,
37
+
deleted: false,
38
+
39
+
subject: spindleMemberRecord.subject,
40
+
instance: spindleMemberRecord.instance,
41
+
createdAt: new Date(spindleMemberRecord.createdAt),
42
+
}).onConflictDoUpdate({
43
+
target: [tangledSpindleMember.did, tangledSpindleMember.rkey],
44
+
set: {
45
+
rev: spindleMember.rev,
46
+
cid: spindleMember.cid,
47
+
deleted: false,
48
+
49
+
subject: spindleMemberRecord.subject,
50
+
instance: spindleMemberRecord.instance,
51
+
createdAt: new Date(spindleMemberRecord.createdAt),
52
+
},
53
+
setWhere: lt(tangledSpindleMember.rev, spindleMember.rev)
54
+
})
55
+
}
56
+
}
+56
api/src/ingest/star.ts
+56
api/src/ingest/star.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconStar from "../lexicons/sh/tangled/feed/star.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledStar } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestStar: CollectionHandler = async (stars) => {
8
+
for (const star of stars) {
9
+
if (star.action === "delete") {
10
+
await db.insert(tangledStar).values({
11
+
did: star.did,
12
+
rev: star.rev,
13
+
rkey: star.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledStar.did, tangledStar.rkey],
18
+
set: {
19
+
rev: star.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledStar.rev, star.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconStar.$safeParse(star.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: starRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledStar).values({
33
+
did: star.did,
34
+
rev: star.rev,
35
+
rkey: star.rkey,
36
+
cid: star.cid,
37
+
deleted: false,
38
+
39
+
subject: starRecord.subject,
40
+
subjectDid: starRecord.subjectDid,
41
+
createdAt: new Date(starRecord.createdAt),
42
+
}).onConflictDoUpdate({
43
+
target: [tangledStar.did, tangledStar.rkey],
44
+
set: {
45
+
rev: star.rev,
46
+
cid: star.cid,
47
+
deleted: false,
48
+
49
+
subject: starRecord.subject,
50
+
subjectDid: starRecord.subjectDid,
51
+
createdAt: new Date(starRecord.createdAt),
52
+
},
53
+
setWhere: lt(tangledStar.rev, star.rev)
54
+
})
55
+
}
56
+
}
+58
api/src/ingest/string.ts
+58
api/src/ingest/string.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconString from "../lexicons/sh/tangled/string.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledString } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestString: CollectionHandler = async (strings) => {
8
+
for (const string of strings) {
9
+
if (string.action === "delete") {
10
+
await db.insert(tangledString).values({
11
+
did: string.did,
12
+
rev: string.rev,
13
+
rkey: string.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledString.did, tangledString.rkey],
18
+
set: {
19
+
rev: string.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledString.rev, string.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconString.$safeParse(string.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: stringRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledString).values({
33
+
did: string.did,
34
+
rev: string.rev,
35
+
rkey: string.rkey,
36
+
cid: string.cid,
37
+
deleted: false,
38
+
39
+
filename: stringRecord.filename,
40
+
description: stringRecord.description,
41
+
createdAt: new Date(stringRecord.createdAt),
42
+
contents: stringRecord.contents,
43
+
}).onConflictDoUpdate({
44
+
target: [tangledString.did, tangledString.rkey],
45
+
set: {
46
+
rev: string.rev,
47
+
cid: string.cid,
48
+
deleted: false,
49
+
50
+
filename: stringRecord.filename,
51
+
description: stringRecord.description,
52
+
createdAt: new Date(stringRecord.createdAt),
53
+
contents: stringRecord.contents,
54
+
},
55
+
setWhere: lt(tangledString.rev, string.rev)
56
+
})
57
+
}
58
+
}
+46
api/src/ingest/tap.ts
+46
api/src/ingest/tap.ts
···
5
5
import { TAP_URL } from '../lib/constants.ts';
6
6
import { ingestProfile } from './profile.ts';
7
7
import { ingestRepo } from './repo.ts';
8
+
import { ingestReaction } from './reaction.ts';
9
+
import { ingestStar } from './star.ts';
10
+
import { ingestRefUpdate } from './refUpdate.ts';
11
+
import { ingestFollow } from './follow.ts';
12
+
import { ingestVouch } from './vouch.ts';
13
+
import { ingestIssueComment } from './issueComment.ts';
14
+
import { ingestIssue } from './issue.ts';
15
+
import { ingestIssueState } from './issueState.ts';
16
+
import { ingestKnot } from './knot.ts';
17
+
import { ingestKnotMember } from './knotMember.ts';
18
+
import { ingestLabelDefinition } from './labelDefinition.ts';
19
+
import { ingestLabelOp } from './labelOp.ts';
20
+
import { ingestPipeline } from './pipeline.ts';
21
+
import { ingestPipelineStatus } from './pipelineStatus.ts';
22
+
import { ingestPublicKey } from './publicKey.ts';
23
+
import { ingestPullComment } from './pullComment.ts';
24
+
import { ingestPull } from './pull.ts';
25
+
import { ingestPullStatus } from './pullStatus.ts';
26
+
import { ingestArtifact } from './artifact.ts';
27
+
import { ingestCollaborator } from './collaborator.ts';
28
+
import { ingestSpindleMember } from './spindleMember.ts';
29
+
import { ingestSpindle } from './spindle.ts';
30
+
import { ingestString } from './string.ts';
8
31
9
32
const tap = new Tap(TAP_URL);
10
33
···
37
60
38
61
const COLLECTION_HANDLERS: Record<string, CollectionHandler> = {
39
62
"sh.tangled.actor.profile": ingestProfile,
63
+
"sh.tangled.feed.reaction": ingestReaction,
64
+
"sh.tangled.feed.star": ingestStar,
65
+
"sh.tangled.git.refUpdate": ingestRefUpdate, // TODO: this actually isn't stored on people's pdses, this comes from knots
66
+
"sh.tangled.graph.follow": ingestFollow,
67
+
"sh.tangled.graph.vouch": ingestVouch,
68
+
"sh.tangled.repo.issue.comment": ingestIssueComment,
69
+
"sh.tangled.repo.issue.state": ingestIssueState,
70
+
"sh.tangled.repo.issue": ingestIssue,
71
+
"sh.tangled.knot": ingestKnot,
72
+
"sh.tangled.knot.member": ingestKnotMember,
73
+
"sh.tangled.label.definition": ingestLabelDefinition,
74
+
"sh.tangled.label.op": ingestLabelOp,
75
+
"sh.tangled.pipeline": ingestPipeline, // TODO: doesn't seem to do anything :(
76
+
"sh.tangled.pipeline.status": ingestPipelineStatus, // TODO: same here :(
77
+
"sh.tangled.publicKey": ingestPublicKey,
78
+
"sh.tangled.repo.pull.comment": ingestPullComment,
79
+
"sh.tangled.repo.pull": ingestPull,
80
+
"sh.tangled.repo.pull.status": ingestPullStatus,
81
+
"sh.tangled.repo.artifact": ingestArtifact,
82
+
"sh.tangled.repo.collaborator": ingestCollaborator,
40
83
"sh.tangled.repo": ingestRepo,
84
+
"sh.tangled.spindle.member": ingestSpindleMember,
85
+
"sh.tangled.spindle": ingestSpindle,
86
+
"sh.tangled.string": ingestString,
41
87
};
42
88
43
89
async function syncLevelTwoTables(insertedRecords: CollectionHandlerArg[], log = false) {
+56
api/src/ingest/vouch.ts
+56
api/src/ingest/vouch.ts
···
1
+
import { lt } from "drizzle-orm";
2
+
import type { CollectionHandler } from "./tap";
3
+
import * as lexiconVouch from "../lexicons/sh/tangled/graph/vouch.ts";
4
+
import { db } from "../db/index.ts";
5
+
import { tangledVouch } from "../db/tables/tangled.ts";
6
+
7
+
export const ingestVouch: CollectionHandler = async (vouches) => {
8
+
for (const vouch of vouches) {
9
+
if (vouch.action === "delete") {
10
+
await db.insert(tangledVouch).values({
11
+
did: vouch.did,
12
+
rev: vouch.rev,
13
+
rkey: vouch.rkey,
14
+
cid: null,
15
+
deleted: true,
16
+
}).onConflictDoUpdate({
17
+
target: [tangledVouch.did, tangledVouch.rkey],
18
+
set: {
19
+
rev: vouch.rev,
20
+
cid: null,
21
+
deleted: true
22
+
},
23
+
setWhere: lt(tangledVouch.rev, vouch.rev)
24
+
});
25
+
continue;
26
+
}
27
+
28
+
const validatedRecord = lexiconVouch.$safeParse(vouch.record);
29
+
if (!validatedRecord.success) continue;
30
+
const { value: vouchRecord } = validatedRecord;
31
+
32
+
await db.insert(tangledVouch).values({
33
+
did: vouch.did,
34
+
rev: vouch.rev,
35
+
rkey: vouch.rkey,
36
+
cid: vouch.cid,
37
+
deleted: false,
38
+
39
+
kind: vouchRecord.kind,
40
+
reason: vouchRecord.reason,
41
+
createdAt: new Date(vouchRecord.createdAt),
42
+
}).onConflictDoUpdate({
43
+
target: [tangledVouch.did, tangledVouch.rkey],
44
+
set: {
45
+
rev: vouch.rev,
46
+
cid: vouch.cid,
47
+
deleted: false,
48
+
49
+
kind: vouchRecord.kind,
50
+
reason: vouchRecord.reason,
51
+
createdAt: new Date(vouchRecord.createdAt),
52
+
},
53
+
setWhere: lt(tangledVouch.rev, vouch.rev)
54
+
})
55
+
}
56
+
}
+8
pnpm-lock.yaml
+8
pnpm-lock.yaml
···
25
25
'@hono/node-server':
26
26
specifier: ^2.0.1
27
27
version: 2.0.1(hono@4.12.16)
28
+
'@std/encoding':
29
+
specifier: jsr:^1.0.10
30
+
version: '@jsr/std__encoding@1.0.10'
28
31
'@std/uuid':
29
32
specifier: jsr:^1.1.1
30
33
version: '@jsr/std__uuid@1.1.1'
···
604
607
605
608
'@jsr/std__crypto@1.1.0':
606
609
resolution: {integrity: sha512-3k5X+V01YqCgIu3tuvS6saFVUn39p+Xe0ITMigwyoDDZsRMUsZcyjwxJpyL6TZF7NFI5Do7alSsHQElZQH1OJg==, tarball: https://npm.jsr.io/~/11/@jsr/std__crypto/1.1.0.tgz}
610
+
611
+
'@jsr/std__encoding@1.0.10':
612
+
resolution: {integrity: sha512-WK2njnDTyKefroRNk2Ooq7GStp6Y0ccAvr4To+Z/zecRAGe7+OSvH9DbiaHpAKwEi2KQbmpWMOYsdNt+TsdmSw==, tarball: https://npm.jsr.io/~/11/@jsr/std__encoding/1.0.10.tgz}
607
613
608
614
'@jsr/std__uuid@1.1.1':
609
615
resolution: {integrity: sha512-LRHNRnFWWC2sGe8bxhDOZgZsomTEBepvjcG/sPmwCGH/4sT0JlcUWoxM6Gf5I01B6BRVMJdEjCY5gPu0buJqWQ==, tarball: https://npm.jsr.io/~/11/@jsr/std__uuid/1.1.1.tgz}
···
1470
1476
'@jsr/std__bytes@1.0.6': {}
1471
1477
1472
1478
'@jsr/std__crypto@1.1.0': {}
1479
+
1480
+
'@jsr/std__encoding@1.0.10': {}
1473
1481
1474
1482
'@jsr/std__uuid@1.1.1':
1475
1483
dependencies: