Fork of github.com/did-method-plc/did-method-plc
Merge pull request #4 from bluesky-social/auditability
Auditability
authored by
Daniel Holmgren
and committed by
GitHub
d92d3f76
3be04b95
+190
-90
+35
-10
packages/lib/src/client.ts
+35
-10
packages/lib/src/client.ts
···
1
-
import { cidForCbor } from '@atproto/common'
1
+
import { check, cidForCbor } from '@atproto/common'
2
2
import { Keypair } from '@atproto/crypto'
3
3
import axios from 'axios'
4
-
import { didForCreateOp, signOperation } from './operations'
4
+
import { didForCreateOp, normalizeOp, signOperation } from './operations'
5
5
import * as t from './types'
6
6
7
7
export class Client {
···
13
13
}
14
14
15
15
async getDocumentData(did: string): Promise<t.DocumentData> {
16
-
const res = await axios.get(`${this.url}/data/${encodeURIComponent(did)}`)
16
+
const res = await axios.get(`${this.url}/${encodeURIComponent(did)}/data`)
17
17
return res.data
18
18
}
19
19
20
-
async getOperationLog(did: string): Promise<t.Operation[]> {
21
-
const res = await axios.get(`${this.url}/log/${encodeURIComponent(did)}`)
22
-
return res.data.log
20
+
async getOperationLog(did: string): Promise<t.CompatibleOpOrTombstone[]> {
21
+
const res = await axios.get(`${this.url}/${encodeURIComponent(did)}/log`)
22
+
return res.data
23
+
}
24
+
25
+
async getAuditableLog(did: string): Promise<t.ExportedOp[]> {
26
+
const res = await axios.get(
27
+
`${this.url}/${encodeURIComponent(did)}/log/audit`,
28
+
)
29
+
return res.data
23
30
}
24
31
25
32
postOpUrl(did: string): string {
26
33
return `${this.url}/${encodeURIComponent(did)}`
27
34
}
28
35
29
-
async getLastOp(did: string): Promise<t.Operation> {
30
-
const res = await axios.get(`${this.url}/last/${encodeURIComponent(did)}`)
36
+
async getLastOp(did: string): Promise<t.CompatibleOpOrTombstone> {
37
+
const res = await axios.get(
38
+
`${this.url}/${encodeURIComponent(did)}/log/last`,
39
+
)
31
40
return res.data
32
41
}
33
42
···
37
46
key: Keypair,
38
47
) {
39
48
const lastOp = await this.getLastOp(did)
49
+
if (check.is(lastOp, t.def.tombstone)) {
50
+
throw new Error('Cannot apply op to tombstone')
51
+
}
40
52
const prev = await cidForCbor(lastOp)
41
-
const { signingKey, rotationKeys, handles, services } = lastOp
53
+
const { signingKey, rotationKeys, handles, services } = normalizeOp(lastOp)
42
54
const op = await signOperation(
43
55
{
44
56
signingKey,
···
69
81
return did
70
82
}
71
83
72
-
async sendOperation(did: string, op: t.Operation) {
84
+
async sendOperation(did: string, op: t.OpOrTombstone) {
73
85
await axios.post(this.postOpUrl(did), op)
86
+
}
87
+
88
+
async export(after?: string, count?: number): Promise<t.ExportedOp[]> {
89
+
const url = new URL(`${this.url}/export`)
90
+
if (after) {
91
+
url.searchParams.append('after', after)
92
+
}
93
+
if (count !== undefined) {
94
+
url.searchParams.append('count', count.toString(10))
95
+
}
96
+
const res = await axios.get(url.toString())
97
+
const lines = res.data.split('\n')
98
+
return lines.map((l) => JSON.parse(l))
74
99
}
75
100
76
101
async health() {
+11
packages/lib/src/types.ts
+11
packages/lib/src/types.ts
···
68
68
})
69
69
export type IndexedOperation = z.infer<typeof indexedOperation>
70
70
71
+
export const exportedOp = z.object({
72
+
did: z.string(),
73
+
operation: compatibleOpOrTombstone,
74
+
cid: z.string(),
75
+
nullified: z.boolean(),
76
+
createdAt: z.string(),
77
+
})
78
+
export type ExportedOp = z.infer<typeof exportedOp>
79
+
71
80
export const didDocVerificationMethod = z.object({
72
81
id: z.string(),
73
82
type: z.string(),
···
102
111
opOrTombstone,
103
112
compatibleOp,
104
113
compatibleOpOrTombstone,
114
+
indexedOperation,
115
+
exportedOp,
105
116
didDocument,
106
117
}
+1
-1
packages/lib/tests/compatibility.test.ts
+1
-1
packages/lib/tests/compatibility.test.ts
+3
-3
packages/lib/tests/recovery.test.ts
+3
-3
packages/lib/tests/recovery.test.ts
···
106
106
expect(res.nullified.length).toBe(2)
107
107
expect(res.nullified[0].equals(log[1].cid))
108
108
expect(res.nullified[1].equals(log[2].cid))
109
-
expect(res.prev?.equals(createCid))
109
+
expect(res.prev?.equals(createCid)).toBeTruthy()
110
110
111
111
log = [log[0], rotate.indexed]
112
112
})
···
124
124
const res = await data.assureValidNextOp(did, log, rotate.op)
125
125
expect(res.nullified.length).toBe(1)
126
126
expect(res.nullified[0].equals(log[1].cid))
127
-
expect(res.prev?.equals(createCid))
127
+
expect(res.prev?.equals(createCid)).toBeTruthy()
128
128
129
129
log = [log[0], rotate.indexed]
130
130
})
···
185
185
)
186
186
expect(result.nullified.length).toBe(1)
187
187
expect(result.nullified[0].equals(cid))
188
-
expect(result.prev?.equals(createCid))
188
+
expect(result.prev?.equals(createCid)).toBeTruthy()
189
189
})
190
190
})
+42
-46
packages/server/src/db/index.ts
+42
-46
packages/server/src/db/index.ts
···
1
-
import { Generated, Kysely, Migrator, PostgresDialect, sql } from 'kysely'
1
+
import { Kysely, Migrator, PostgresDialect, sql } from 'kysely'
2
2
import { Pool as PgPool, types as pgTypes } from 'pg'
3
3
import { CID } from 'multiformats/cid'
4
-
import { cidForCbor, check } from '@atproto/common'
4
+
import { cidForCbor } from '@atproto/common'
5
5
import * as plc from '@did-plc/lib'
6
6
import { ServerError } from '../error'
7
7
import * as migrations from '../migrations'
8
-
import { OpLogExport, PlcDatabase } from './types'
8
+
import { DatabaseSchema, PlcDatabase } from './types'
9
9
import MockDatabase from './mock'
10
10
11
11
export * from './mock'
···
93
93
}
94
94
95
95
async validateAndAddOp(did: string, proposed: plc.Operation): Promise<void> {
96
-
const ops = await this._opsForDid(did)
96
+
const ops = await this.indexedOpsForDid(did)
97
97
// throws if invalid
98
98
const { nullified, prev } = await plc.assureValidNextOp(did, ops, proposed)
99
99
const cid = await cidForCbor(proposed)
···
158
158
return found ? CID.parse(found.cid) : null
159
159
}
160
160
161
-
async opsForDid(did: string): Promise<plc.OpOrTombstone[]> {
162
-
const ops = await this._opsForDid(did)
163
-
return ops.map((op) => {
164
-
if (check.is(op.operation, plc.def.createOpV1)) {
165
-
return plc.normalizeOp(op.operation)
166
-
}
167
-
return op.operation
168
-
})
161
+
async opsForDid(did: string): Promise<plc.CompatibleOpOrTombstone[]> {
162
+
const ops = await this.indexedOpsForDid(did)
163
+
return ops.map((op) => op.operation)
169
164
}
170
165
171
-
async _opsForDid(did: string): Promise<plc.IndexedOperation[]> {
172
-
const res = await this.db
166
+
async indexedOpsForDid(
167
+
did: string,
168
+
includeNullified = false,
169
+
): Promise<plc.IndexedOperation[]> {
170
+
let builder = this.db
173
171
.selectFrom('operations')
174
172
.selectAll()
175
173
.where('did', '=', did)
176
-
.where('nullified', '=', false)
177
174
.orderBy('createdAt', 'asc')
178
-
.execute()
179
-
175
+
if (!includeNullified) {
176
+
builder = builder.where('nullified', '=', false)
177
+
}
178
+
const res = await builder.execute()
180
179
return res.map((row) => ({
181
180
did: row.did,
182
181
operation: row.operation,
···
186
185
}))
187
186
}
188
187
189
-
async fullExport(): Promise<Record<string, OpLogExport>> {
190
-
return {}
191
-
// const res = await this.db
192
-
// .selectFrom('operations')
193
-
// .selectAll()
194
-
// .orderBy('did')
195
-
// .orderBy('createdAt')
196
-
// .execute()
197
-
// return res.reduce((acc, cur) => {
198
-
// acc[cur.did] ??= []
199
-
// acc[cur.did].push({
200
-
// op: cur.operation),
201
-
// nullified: cur.nullified === 1,
202
-
// createdAt: cur.createdAt,
203
-
// })
204
-
// return acc
205
-
// }, {} as Record<string, OpLogExport>)
188
+
async lastOpForDid(did: string): Promise<plc.CompatibleOpOrTombstone | null> {
189
+
const res = await this.db
190
+
.selectFrom('operations')
191
+
.selectAll()
192
+
.where('did', '=', did)
193
+
.where('nullified', '=', false)
194
+
.orderBy('createdAt', 'desc')
195
+
.limit(1)
196
+
.executeTakeFirst()
197
+
return res?.operation ?? null
206
198
}
207
-
}
208
199
209
-
export default Database
210
-
211
-
interface OperationsTable {
212
-
did: string
213
-
operation: plc.CompatibleOpOrTombstone
214
-
cid: string
215
-
nullified: boolean
216
-
createdAt: Generated<Date>
200
+
async exportOps(count: number, after?: Date): Promise<plc.ExportedOp[]> {
201
+
let builder = this.db
202
+
.selectFrom('operations')
203
+
.selectAll()
204
+
.orderBy('createdAt', 'asc')
205
+
.limit(count)
206
+
if (after) {
207
+
builder = builder.where('createdAt', '>', after)
208
+
}
209
+
const res = await builder.execute()
210
+
return res.map((row) => ({
211
+
...row,
212
+
createdAt: row.createdAt.toISOString(),
213
+
}))
214
+
}
217
215
}
218
216
219
-
interface DatabaseSchema {
220
-
operations: OperationsTable
221
-
}
217
+
export default Database
+22
-7
packages/server/src/db/mock.ts
+22
-7
packages/server/src/db/mock.ts
···
1
1
import { cidForCbor, check } from '@atproto/common'
2
2
import * as plc from '@did-plc/lib'
3
3
import { ServerError } from '../error'
4
-
import { OpLogExport, PlcDatabase } from './types'
4
+
import { PlcDatabase } from './types'
5
5
6
6
type Contents = Record<string, plc.IndexedOperation[]>
7
7
···
43
43
}
44
44
}
45
45
46
-
async opsForDid(did: string): Promise<plc.OpOrTombstone[]> {
47
-
const ops = await this._opsForDid(did)
46
+
async opsForDid(did: string): Promise<plc.CompatibleOpOrTombstone[]> {
47
+
const ops = await this.indexedOpsForDid(did)
48
48
return ops.map((op) => {
49
49
if (check.is(op.operation, plc.def.createOpV1)) {
50
50
return plc.normalizeOp(op.operation)
···
53
53
})
54
54
}
55
55
56
-
async _opsForDid(did: string): Promise<plc.IndexedOperation[]> {
57
-
return this.contents[did] ?? []
56
+
async indexedOpsForDid(
57
+
did: string,
58
+
includeNull = false,
59
+
): Promise<plc.IndexedOperation[]> {
60
+
const ops = this.contents[did] ?? []
61
+
if (includeNull) {
62
+
return ops
63
+
}
64
+
return ops.filter((op) => op.nullified === false)
65
+
}
66
+
67
+
async lastOpForDid(did: string): Promise<plc.CompatibleOpOrTombstone | null> {
68
+
const op = this.contents[did]?.at(-1)
69
+
70
+
if (!op) return null
71
+
return op.operation
58
72
}
59
73
60
-
async fullExport(): Promise<Record<string, OpLogExport>> {
61
-
return {}
74
+
// disabled in mocks
75
+
async exportOps(_count: number, _after?: Date): Promise<plc.ExportedOp[]> {
76
+
return []
62
77
}
63
78
}
64
79
+17
-8
packages/server/src/db/types.ts
+17
-8
packages/server/src/db/types.ts
···
1
1
import * as plc from '@did-plc/lib'
2
+
import { Generated } from 'kysely'
2
3
3
4
export interface PlcDatabase {
4
5
close(): Promise<void>
5
6
healthCheck(): Promise<void>
6
7
validateAndAddOp(did: string, proposed: plc.Operation): Promise<void>
7
-
opsForDid(did: string): Promise<plc.OpOrTombstone[]>
8
-
_opsForDid(did: string): Promise<plc.IndexedOperation[]>
9
-
fullExport(): Promise<Record<string, OpLogExport>>
8
+
opsForDid(did: string): Promise<plc.CompatibleOpOrTombstone[]>
9
+
indexedOpsForDid(
10
+
did: string,
11
+
includeNull?: boolean,
12
+
): Promise<plc.IndexedOperation[]>
13
+
lastOpForDid(did: string): Promise<plc.CompatibleOpOrTombstone | null>
14
+
exportOps(count: number, after?: Date): Promise<plc.ExportedOp[]>
10
15
}
11
16
12
-
export type OpLogExport = OpExport[]
13
-
14
-
export type OpExport = {
15
-
op: Record<string, unknown>
17
+
export interface OperationsTable {
18
+
did: string
19
+
operation: plc.CompatibleOpOrTombstone
20
+
cid: string
16
21
nullified: boolean
17
-
createdAt: string
22
+
createdAt: Generated<Date>
23
+
}
24
+
25
+
export interface DatabaseSchema {
26
+
operations: OperationsTable
18
27
}
+36
-13
packages/server/src/routes.ts
+36
-13
packages/server/src/routes.ts
···
18
18
res.send({ version })
19
19
})
20
20
21
-
// @TODO paginate & test this
21
+
// Export ops in the form of paginated json lines
22
22
router.get('/export', async function (req, res) {
23
-
const fullExport = await ctx.db.fullExport()
23
+
const parsedCount = req.count ? parseInt(req.count, 10) : 1000
24
+
if (isNaN(parsedCount) || parsedCount < 1) {
25
+
throw new ServerError(400, 'Invalid count parameter')
26
+
}
27
+
const count = Math.min(parsedCount, 1000)
28
+
const after = req.query.after ? new Date(req.query.after) : undefined
29
+
const ops = await ctx.db.exportOps(count, after)
24
30
res.setHeader('content-type', 'application/jsonlines')
25
31
res.status(200)
26
-
for (const [did, ops] of Object.entries(fullExport)) {
27
-
const line = JSON.stringify({ did, ops })
32
+
for (let i = 0; i < ops.length; i++) {
33
+
if (i > 0) {
34
+
res.write('\n')
35
+
}
36
+
const line = JSON.stringify(ops[i])
28
37
res.write(line)
29
-
res.write('\n')
30
38
}
31
39
res.end()
32
40
})
···
48
56
})
49
57
50
58
// Get data for a DID document
51
-
router.get('/data/:did', async function (req, res) {
59
+
router.get('/:did/data', async function (req, res) {
52
60
const { did } = req.params
53
61
const log = await ctx.db.opsForDid(did)
54
62
if (log.length === 0) {
···
62
70
})
63
71
64
72
// Get operation log for a DID
65
-
router.get('/log/:did', async function (req, res) {
73
+
router.get('/:did/log', async function (req, res) {
66
74
const { did } = req.params
67
75
const log = await ctx.db.opsForDid(did)
68
76
if (log.length === 0) {
69
77
throw new ServerError(404, `DID not registered: ${did}`)
70
78
}
71
-
res.json({ log })
79
+
res.json(log)
80
+
})
81
+
82
+
// Get operation log for a DID
83
+
router.get('/:did/log/audit', async function (req, res) {
84
+
const { did } = req.params
85
+
const ops = await ctx.db.indexedOpsForDid(did, true)
86
+
if (ops.length === 0) {
87
+
throw new ServerError(404, `DID not registered: ${did}`)
88
+
}
89
+
const log = ops.map((op) => ({
90
+
...op,
91
+
cid: op.cid.toString(),
92
+
createdAt: op.createdAt.toISOString(),
93
+
}))
94
+
95
+
res.json(log)
72
96
})
73
97
74
98
// Get the most recent operation in the log for a DID
75
-
router.get('/last/:did', async function (req, res) {
99
+
router.get('/:did/log/last', async function (req, res) {
76
100
const { did } = req.params
77
-
const log = await ctx.db.opsForDid(did)
78
-
const curr = log.at(-1)
79
-
if (!curr) {
101
+
const last = await ctx.db.lastOpForDid(did)
102
+
if (!last) {
80
103
throw new ServerError(404, `DID not registered: ${did}`)
81
104
}
82
-
res.json(curr)
105
+
res.json(last)
83
106
})
84
107
85
108
// Update or create a DID doc
+23
-2
packages/server/tests/server.test.ts
+23
-2
packages/server/tests/server.test.ts
···
1
1
import { EcdsaKeypair } from '@atproto/crypto'
2
2
import * as plc from '@did-plc/lib'
3
3
import { CloseFn, runTestServer } from './_util'
4
-
import { cidForCbor } from '@atproto/common'
4
+
import { check, cidForCbor } from '@atproto/common'
5
5
import { AxiosError } from 'axios'
6
6
import { Database } from '../src'
7
7
import { signOperation } from '@did-plc/lib'
···
133
133
const newKey = await EcdsaKeypair.create()
134
134
const ops = await client.getOperationLog(did)
135
135
const forkPoint = ops.at(-2)
136
-
if (!forkPoint) {
136
+
if (!check.is(forkPoint, plc.def.operation)) {
137
137
throw new Error('Could not find fork point')
138
138
}
139
139
const forkCid = await cidForCbor(forkPoint)
···
157
157
expect(doc.rotationKeys).toEqual([newKey.did()])
158
158
expect(doc.handles).toEqual([handle])
159
159
expect(doc.services).toEqual({ atpPds })
160
+
})
161
+
162
+
it('retrieves the auditable operation log', async () => {
163
+
const log = await client.getOperationLog(did)
164
+
const auditable = await client.getAuditableLog(did)
165
+
// has one nullifed op
166
+
expect(auditable.length).toBe(log.length + 1)
167
+
expect(auditable.filter((op) => op.nullified).length).toBe(1)
168
+
expect(auditable.at(-2)?.nullified).toBe(true)
169
+
expect(
170
+
auditable.every((op) => check.is(op, plc.def.exportedOp)),
171
+
).toBeTruthy()
160
172
})
161
173
162
174
it('retrieves the did doc', async () => {
···
217
229
218
230
const ops = await client.getOperationLog(did)
219
231
await plc.validateOperationLog(did, ops)
232
+
})
233
+
234
+
it('exports the data set', async () => {
235
+
const data = await client.export()
236
+
expect(data.every((row) => check.is(row, plc.def.exportedOp))).toBeTruthy()
237
+
expect(data.length).toBe(58)
238
+
for (let i = 1; i < data.length; i++) {
239
+
expect(data[i].createdAt >= data[i - 1].createdAt).toBeTruthy()
240
+
}
220
241
})
221
242
222
243
it('healthcheck succeeds when database is available.', async () => {