this repo has no description
atproto/data: package for working with generic schema-less atproto data in JSON or CBOR (#407)
So far most golang atproto code has either worked with data in a known
schema, or entirely ignored the contents. We will at times need to
convert to and from JSON and CBOR when a schema isn't available. Or even
when the schema is available, a record might have unexpected additional
fields, and conversion might need to be done in a way that preserves all
fields.
Reused some existing test fixtures to ensure interop and consistent
behavior against typescript implementation.
The Bytes, Blob, and CIDLink structs are copied from `lex/util`, and
could replace those implementations.
Expect to implement `atproto/lexicon` on top of this package to do
run-time schema validation.
"Extract all blobs from unknown data" is a thing that the distributor or
appview v2 may need to do in the near future, to copy in to CDN and run
image auto-moderation.
I bet there is a much more efficient way to implement the "parsing"
functions. The goal with this initial version of the package is to get a
reasonable API and test coverage of weird corner-cases. Should be able
to optimize the implementation later.
+1708
-1
+1
-1
.gitignore
+1
-1
.gitignore
+87
atproto/data/basic_test.go
+87
atproto/data/basic_test.go
···
1
+
package data
2
+
3
+
import (
4
+
"encoding/json"
5
+
"testing"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
9
+
"github.com/ipfs/go-cid"
10
+
"github.com/stretchr/testify/assert"
11
+
)
12
+
13
+
func TestSimpleValidation(t *testing.T) {
14
+
assert := assert.New(t)
15
+
16
+
s := "a string"
17
+
assert.NoError(Validate(map[string]interface{}{
18
+
"a": 5,
19
+
"b": 123,
20
+
"c": s,
21
+
"d": &s,
22
+
}))
23
+
assert.NoError(Validate(map[string]interface{}{
24
+
"$type": "com.example.thing",
25
+
"a": 5,
26
+
}))
27
+
assert.Error(Validate(map[string]interface{}{
28
+
"$type": 123,
29
+
"a": 5,
30
+
}))
31
+
assert.Error(Validate(map[string]interface{}{
32
+
"$type": "",
33
+
"a": 5,
34
+
}))
35
+
}
36
+
37
+
func TestSyntaxSerialize(t *testing.T) {
38
+
assert := assert.New(t)
39
+
40
+
atid, err := syntax.ParseAtIdentifier("did:web:example.com")
41
+
assert.NoError(err)
42
+
obj := map[string]interface{}{
43
+
"at-identifier": atid,
44
+
"at-uri": syntax.ATURI("at://did:abc:123/io.nsid.someFunc/record-key"),
45
+
"cid-string": syntax.CID("bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"),
46
+
"datetime": syntax.Datetime("2023-10-30T22:25:23Z"),
47
+
"did": syntax.DID("did:web:example.com"),
48
+
"handle": syntax.Handle("blah.example.com"),
49
+
"language": syntax.Language("us"),
50
+
"nsid": syntax.NSID("com.example.blah"),
51
+
"recordkey": syntax.RecordKey("self"),
52
+
"tid": syntax.TID("3kao2cl6lyj2p"),
53
+
"uri": syntax.URI("https://example.com/file"),
54
+
}
55
+
assert.NoError(Validate(obj))
56
+
_, err = MarshalCBOR(obj)
57
+
assert.NoError(err)
58
+
_, err = json.Marshal(obj)
59
+
assert.NoError(err)
60
+
}
61
+
62
+
func TestExtractBlobs(t *testing.T) {
63
+
assert := assert.New(t)
64
+
65
+
cid1, _ := cid.Parse("bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity")
66
+
obj := map[string]interface{}{
67
+
"a": 5,
68
+
"b": 123,
69
+
"c": map[string]interface{}{
70
+
"blb": Blob{
71
+
Size: 567,
72
+
MimeType: "image/jpeg",
73
+
Ref: CIDLink(cid1),
74
+
},
75
+
},
76
+
"d": []interface{}{
77
+
123,
78
+
Blob{
79
+
Size: 123,
80
+
MimeType: "image/png",
81
+
Ref: CIDLink(cid1),
82
+
},
83
+
},
84
+
}
85
+
blbs := ExtractBlobs(obj)
86
+
assert.Equal(2, len(blbs))
87
+
}
+146
atproto/data/blob.go
+146
atproto/data/blob.go
···
1
+
package data
2
+
3
+
import (
4
+
"bytes"
5
+
"encoding/json"
6
+
"fmt"
7
+
"io"
8
+
9
+
"github.com/ipfs/go-cid"
10
+
cbg "github.com/whyrusleeping/cbor-gen"
11
+
)
12
+
13
+
// Represents the "blob" type from the atproto data model.
14
+
//
15
+
// This struct does not get marshaled/unmarshaled directly in to JSON or CBOR; see the BlobSchema and LegacyBlobSchema structs. This is the type that should be included in golang struct definitions.
16
+
//
17
+
// When representing a "legacy" blob (no size field, string CID), size == -1.
18
+
type Blob struct {
19
+
Ref CIDLink
20
+
MimeType string
21
+
Size int64
22
+
}
23
+
24
+
type LegacyBlobSchema struct {
25
+
Cid string `json:"cid" cborgen:"cid"`
26
+
MimeType string `json:"mimeType" cborgen:"mimeType"`
27
+
}
28
+
29
+
type BlobSchema struct {
30
+
LexiconTypeID string `json:"$type,const=blob" cborgen:"$type,const=blob"`
31
+
Ref CIDLink `json:"ref" cborgen:"ref"`
32
+
MimeType string `json:"mimeType" cborgen:"mimeType"`
33
+
Size int64 `json:"size" cborgen:"size"`
34
+
}
35
+
36
+
func (b Blob) MarshalJSON() ([]byte, error) {
37
+
if b.Size < 0 {
38
+
lb := LegacyBlobSchema{
39
+
Cid: b.Ref.String(),
40
+
MimeType: b.MimeType,
41
+
}
42
+
return json.Marshal(lb)
43
+
} else {
44
+
nb := BlobSchema{
45
+
LexiconTypeID: "blob",
46
+
Ref: b.Ref,
47
+
MimeType: b.MimeType,
48
+
Size: b.Size,
49
+
}
50
+
return json.Marshal(nb)
51
+
}
52
+
}
53
+
54
+
func (b *Blob) UnmarshalJSON(raw []byte) error {
55
+
typ, err := ExtractTypeJSON(raw)
56
+
if err != nil {
57
+
return fmt.Errorf("parsing blob type: %v", err)
58
+
}
59
+
60
+
if typ == "blob" {
61
+
var bs BlobSchema
62
+
err := json.Unmarshal(raw, &bs)
63
+
if err != nil {
64
+
return fmt.Errorf("parsing blob JSON: %v", err)
65
+
}
66
+
b.Ref = bs.Ref
67
+
b.MimeType = bs.MimeType
68
+
b.Size = bs.Size
69
+
if bs.Size < 0 {
70
+
return fmt.Errorf("parsing blob: negative size: %d", bs.Size)
71
+
}
72
+
} else {
73
+
var legacy LegacyBlobSchema
74
+
err := json.Unmarshal(raw, &legacy)
75
+
if err != nil {
76
+
return fmt.Errorf("parsing legacy blob: %v", err)
77
+
}
78
+
refCid, err := cid.Decode(legacy.Cid)
79
+
if err != nil {
80
+
return fmt.Errorf("parsing CID in legacy blob: %v", err)
81
+
}
82
+
b.Ref = CIDLink(refCid)
83
+
b.MimeType = legacy.MimeType
84
+
b.Size = -1
85
+
}
86
+
return nil
87
+
}
88
+
89
+
func (b *Blob) MarshalCBOR(w io.Writer) error {
90
+
if b == nil {
91
+
_, err := w.Write(cbg.CborNull)
92
+
return err
93
+
}
94
+
if b.Size < 0 {
95
+
lb := LegacyBlobSchema{
96
+
Cid: b.Ref.String(),
97
+
MimeType: b.MimeType,
98
+
}
99
+
return lb.MarshalCBOR(w)
100
+
} else {
101
+
bs := BlobSchema{
102
+
LexiconTypeID: "blob",
103
+
Ref: b.Ref,
104
+
MimeType: b.MimeType,
105
+
Size: b.Size,
106
+
}
107
+
return bs.MarshalCBOR(w)
108
+
}
109
+
}
110
+
111
+
func (lb *Blob) UnmarshalCBOR(r io.Reader) error {
112
+
typ, b, err := ExtractTypeCBORReader(r)
113
+
if err != nil {
114
+
return fmt.Errorf("parsing $blob CBOR type: %w", err)
115
+
}
116
+
117
+
*lb = Blob{}
118
+
if typ == "blob" {
119
+
var bs BlobSchema
120
+
err := bs.UnmarshalCBOR(bytes.NewReader(b))
121
+
if err != nil {
122
+
return fmt.Errorf("parsing $blob CBOR: %v", err)
123
+
}
124
+
lb.Ref = bs.Ref
125
+
lb.MimeType = bs.MimeType
126
+
lb.Size = bs.Size
127
+
if bs.Size < 0 {
128
+
return fmt.Errorf("parsing $blob CBOR: negative size: %d", bs.Size)
129
+
}
130
+
} else {
131
+
legacy := LegacyBlobSchema{}
132
+
err := legacy.UnmarshalCBOR(bytes.NewReader(b))
133
+
if err != nil {
134
+
return fmt.Errorf("parsing legacy blob CBOR: %v", err)
135
+
}
136
+
refCid, err := cid.Decode(legacy.Cid)
137
+
if err != nil {
138
+
return fmt.Errorf("parsing CID in legacy blob CBOR: %v", err)
139
+
}
140
+
lb.Ref = CIDLink(refCid)
141
+
lb.MimeType = legacy.MimeType
142
+
lb.Size = -1
143
+
}
144
+
145
+
return nil
146
+
}
+67
atproto/data/bytes.go
+67
atproto/data/bytes.go
···
1
+
package data
2
+
3
+
import (
4
+
"encoding/base64"
5
+
"encoding/json"
6
+
"fmt"
7
+
"io"
8
+
9
+
cbg "github.com/whyrusleeping/cbor-gen"
10
+
)
11
+
12
+
// Represents the "bytes" type from the atproto data model.
13
+
//
14
+
// In JSON, marshals to an object with $bytes key and base64-encoded data.
15
+
//
16
+
// In CBOR, marshals to a byte array.
17
+
type Bytes []byte
18
+
19
+
type JsonBytes struct {
20
+
Bytes string `json:"$bytes"`
21
+
}
22
+
23
+
func (lb Bytes) MarshalJSON() ([]byte, error) {
24
+
if lb == nil {
25
+
return nil, fmt.Errorf("tried to marshal nil $bytes")
26
+
}
27
+
jb := JsonBytes{
28
+
Bytes: base64.RawStdEncoding.EncodeToString([]byte(lb)),
29
+
}
30
+
return json.Marshal(jb)
31
+
}
32
+
33
+
func (lb *Bytes) UnmarshalJSON(raw []byte) error {
34
+
var jb JsonBytes
35
+
err := json.Unmarshal(raw, &jb)
36
+
if err != nil {
37
+
return fmt.Errorf("parsing $bytes JSON: %v", err)
38
+
}
39
+
out, err := base64.RawStdEncoding.DecodeString(jb.Bytes)
40
+
if err != nil {
41
+
return fmt.Errorf("parsing $bytes base64: %v", err)
42
+
}
43
+
*lb = Bytes(out)
44
+
return nil
45
+
}
46
+
47
+
func (lb *Bytes) MarshalCBOR(w io.Writer) error {
48
+
if lb == nil {
49
+
_, err := w.Write(cbg.CborNull)
50
+
return err
51
+
}
52
+
cw := cbg.NewCborWriter(w)
53
+
if err := cbg.WriteByteArray(cw, ([]byte)(*lb)); err != nil {
54
+
return fmt.Errorf("failed to write $bytes as CBOR: %w", err)
55
+
}
56
+
return nil
57
+
}
58
+
59
+
func (lb *Bytes) UnmarshalCBOR(r io.Reader) error {
60
+
cr := cbg.NewCborReader(r)
61
+
b, err := cbg.ReadByteArray(cr, MAX_RECORD_BYTES_LEN)
62
+
if err != nil {
63
+
return fmt.Errorf("failed to read $bytes from CBOR: %w", err)
64
+
}
65
+
*lb = Bytes(b)
66
+
return nil
67
+
}
+443
atproto/data/cbor_gen.go
+443
atproto/data/cbor_gen.go
···
1
+
// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT.
2
+
3
+
package data
4
+
5
+
import (
6
+
"fmt"
7
+
"io"
8
+
"math"
9
+
"sort"
10
+
11
+
cid "github.com/ipfs/go-cid"
12
+
cbg "github.com/whyrusleeping/cbor-gen"
13
+
xerrors "golang.org/x/xerrors"
14
+
)
15
+
16
+
var _ = xerrors.Errorf
17
+
var _ = cid.Undef
18
+
var _ = math.E
19
+
var _ = sort.Sort
20
+
21
+
func (t *GenericRecord) MarshalCBOR(w io.Writer) error {
22
+
if t == nil {
23
+
_, err := w.Write(cbg.CborNull)
24
+
return err
25
+
}
26
+
27
+
cw := cbg.NewCborWriter(w)
28
+
29
+
if _, err := cw.Write([]byte{161}); err != nil {
30
+
return err
31
+
}
32
+
33
+
// t.Type (string) (string)
34
+
if len("$type") > cbg.MaxLength {
35
+
return xerrors.Errorf("Value in field \"$type\" was too long")
36
+
}
37
+
38
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil {
39
+
return err
40
+
}
41
+
if _, err := cw.WriteString(string("$type")); err != nil {
42
+
return err
43
+
}
44
+
45
+
if len(t.Type) > cbg.MaxLength {
46
+
return xerrors.Errorf("Value in field t.Type was too long")
47
+
}
48
+
49
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Type))); err != nil {
50
+
return err
51
+
}
52
+
if _, err := cw.WriteString(string(t.Type)); err != nil {
53
+
return err
54
+
}
55
+
return nil
56
+
}
57
+
58
+
func (t *GenericRecord) UnmarshalCBOR(r io.Reader) (err error) {
59
+
*t = GenericRecord{}
60
+
61
+
cr := cbg.NewCborReader(r)
62
+
63
+
maj, extra, err := cr.ReadHeader()
64
+
if err != nil {
65
+
return err
66
+
}
67
+
defer func() {
68
+
if err == io.EOF {
69
+
err = io.ErrUnexpectedEOF
70
+
}
71
+
}()
72
+
73
+
if maj != cbg.MajMap {
74
+
return fmt.Errorf("cbor input should be of type map")
75
+
}
76
+
77
+
if extra > cbg.MaxLength {
78
+
return fmt.Errorf("GenericRecord: map struct too large (%d)", extra)
79
+
}
80
+
81
+
var name string
82
+
n := extra
83
+
84
+
for i := uint64(0); i < n; i++ {
85
+
86
+
{
87
+
sval, err := cbg.ReadString(cr)
88
+
if err != nil {
89
+
return err
90
+
}
91
+
92
+
name = string(sval)
93
+
}
94
+
95
+
switch name {
96
+
// t.Type (string) (string)
97
+
case "$type":
98
+
99
+
{
100
+
sval, err := cbg.ReadString(cr)
101
+
if err != nil {
102
+
return err
103
+
}
104
+
105
+
t.Type = string(sval)
106
+
}
107
+
108
+
default:
109
+
// Field doesn't exist on this type, so ignore it
110
+
cbg.ScanForLinks(r, func(cid.Cid) {})
111
+
}
112
+
}
113
+
114
+
return nil
115
+
}
116
+
func (t *LegacyBlobSchema) MarshalCBOR(w io.Writer) error {
117
+
if t == nil {
118
+
_, err := w.Write(cbg.CborNull)
119
+
return err
120
+
}
121
+
122
+
cw := cbg.NewCborWriter(w)
123
+
124
+
if _, err := cw.Write([]byte{162}); err != nil {
125
+
return err
126
+
}
127
+
128
+
// t.Cid (string) (string)
129
+
if len("cid") > cbg.MaxLength {
130
+
return xerrors.Errorf("Value in field \"cid\" was too long")
131
+
}
132
+
133
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil {
134
+
return err
135
+
}
136
+
if _, err := cw.WriteString(string("cid")); err != nil {
137
+
return err
138
+
}
139
+
140
+
if len(t.Cid) > cbg.MaxLength {
141
+
return xerrors.Errorf("Value in field t.Cid was too long")
142
+
}
143
+
144
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cid))); err != nil {
145
+
return err
146
+
}
147
+
if _, err := cw.WriteString(string(t.Cid)); err != nil {
148
+
return err
149
+
}
150
+
151
+
// t.MimeType (string) (string)
152
+
if len("mimeType") > cbg.MaxLength {
153
+
return xerrors.Errorf("Value in field \"mimeType\" was too long")
154
+
}
155
+
156
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mimeType"))); err != nil {
157
+
return err
158
+
}
159
+
if _, err := cw.WriteString(string("mimeType")); err != nil {
160
+
return err
161
+
}
162
+
163
+
if len(t.MimeType) > cbg.MaxLength {
164
+
return xerrors.Errorf("Value in field t.MimeType was too long")
165
+
}
166
+
167
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MimeType))); err != nil {
168
+
return err
169
+
}
170
+
if _, err := cw.WriteString(string(t.MimeType)); err != nil {
171
+
return err
172
+
}
173
+
return nil
174
+
}
175
+
176
+
func (t *LegacyBlobSchema) UnmarshalCBOR(r io.Reader) (err error) {
177
+
*t = LegacyBlobSchema{}
178
+
179
+
cr := cbg.NewCborReader(r)
180
+
181
+
maj, extra, err := cr.ReadHeader()
182
+
if err != nil {
183
+
return err
184
+
}
185
+
defer func() {
186
+
if err == io.EOF {
187
+
err = io.ErrUnexpectedEOF
188
+
}
189
+
}()
190
+
191
+
if maj != cbg.MajMap {
192
+
return fmt.Errorf("cbor input should be of type map")
193
+
}
194
+
195
+
if extra > cbg.MaxLength {
196
+
return fmt.Errorf("LegacyBlobSchema: map struct too large (%d)", extra)
197
+
}
198
+
199
+
var name string
200
+
n := extra
201
+
202
+
for i := uint64(0); i < n; i++ {
203
+
204
+
{
205
+
sval, err := cbg.ReadString(cr)
206
+
if err != nil {
207
+
return err
208
+
}
209
+
210
+
name = string(sval)
211
+
}
212
+
213
+
switch name {
214
+
// t.Cid (string) (string)
215
+
case "cid":
216
+
217
+
{
218
+
sval, err := cbg.ReadString(cr)
219
+
if err != nil {
220
+
return err
221
+
}
222
+
223
+
t.Cid = string(sval)
224
+
}
225
+
// t.MimeType (string) (string)
226
+
case "mimeType":
227
+
228
+
{
229
+
sval, err := cbg.ReadString(cr)
230
+
if err != nil {
231
+
return err
232
+
}
233
+
234
+
t.MimeType = string(sval)
235
+
}
236
+
237
+
default:
238
+
// Field doesn't exist on this type, so ignore it
239
+
cbg.ScanForLinks(r, func(cid.Cid) {})
240
+
}
241
+
}
242
+
243
+
return nil
244
+
}
245
+
func (t *BlobSchema) MarshalCBOR(w io.Writer) error {
246
+
if t == nil {
247
+
_, err := w.Write(cbg.CborNull)
248
+
return err
249
+
}
250
+
251
+
cw := cbg.NewCborWriter(w)
252
+
253
+
if _, err := cw.Write([]byte{164}); err != nil {
254
+
return err
255
+
}
256
+
257
+
// t.Ref (data.CIDLink) (struct)
258
+
if len("ref") > cbg.MaxLength {
259
+
return xerrors.Errorf("Value in field \"ref\" was too long")
260
+
}
261
+
262
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ref"))); err != nil {
263
+
return err
264
+
}
265
+
if _, err := cw.WriteString(string("ref")); err != nil {
266
+
return err
267
+
}
268
+
269
+
if err := t.Ref.MarshalCBOR(cw); err != nil {
270
+
return err
271
+
}
272
+
273
+
// t.Size (int64) (int64)
274
+
if len("size") > cbg.MaxLength {
275
+
return xerrors.Errorf("Value in field \"size\" was too long")
276
+
}
277
+
278
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("size"))); err != nil {
279
+
return err
280
+
}
281
+
if _, err := cw.WriteString(string("size")); err != nil {
282
+
return err
283
+
}
284
+
285
+
if t.Size >= 0 {
286
+
if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Size)); err != nil {
287
+
return err
288
+
}
289
+
} else {
290
+
if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Size-1)); err != nil {
291
+
return err
292
+
}
293
+
}
294
+
295
+
// t.LexiconTypeID (string) (string)
296
+
if len("$type") > cbg.MaxLength {
297
+
return xerrors.Errorf("Value in field \"$type\" was too long")
298
+
}
299
+
300
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil {
301
+
return err
302
+
}
303
+
if _, err := cw.WriteString(string("$type")); err != nil {
304
+
return err
305
+
}
306
+
307
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blob"))); err != nil {
308
+
return err
309
+
}
310
+
if _, err := cw.WriteString(string("blob")); err != nil {
311
+
return err
312
+
}
313
+
314
+
// t.MimeType (string) (string)
315
+
if len("mimeType") > cbg.MaxLength {
316
+
return xerrors.Errorf("Value in field \"mimeType\" was too long")
317
+
}
318
+
319
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mimeType"))); err != nil {
320
+
return err
321
+
}
322
+
if _, err := cw.WriteString(string("mimeType")); err != nil {
323
+
return err
324
+
}
325
+
326
+
if len(t.MimeType) > cbg.MaxLength {
327
+
return xerrors.Errorf("Value in field t.MimeType was too long")
328
+
}
329
+
330
+
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MimeType))); err != nil {
331
+
return err
332
+
}
333
+
if _, err := cw.WriteString(string(t.MimeType)); err != nil {
334
+
return err
335
+
}
336
+
return nil
337
+
}
338
+
339
+
func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) {
340
+
*t = BlobSchema{}
341
+
342
+
cr := cbg.NewCborReader(r)
343
+
344
+
maj, extra, err := cr.ReadHeader()
345
+
if err != nil {
346
+
return err
347
+
}
348
+
defer func() {
349
+
if err == io.EOF {
350
+
err = io.ErrUnexpectedEOF
351
+
}
352
+
}()
353
+
354
+
if maj != cbg.MajMap {
355
+
return fmt.Errorf("cbor input should be of type map")
356
+
}
357
+
358
+
if extra > cbg.MaxLength {
359
+
return fmt.Errorf("BlobSchema: map struct too large (%d)", extra)
360
+
}
361
+
362
+
var name string
363
+
n := extra
364
+
365
+
for i := uint64(0); i < n; i++ {
366
+
367
+
{
368
+
sval, err := cbg.ReadString(cr)
369
+
if err != nil {
370
+
return err
371
+
}
372
+
373
+
name = string(sval)
374
+
}
375
+
376
+
switch name {
377
+
// t.Ref (data.CIDLink) (struct)
378
+
case "ref":
379
+
380
+
{
381
+
382
+
if err := t.Ref.UnmarshalCBOR(cr); err != nil {
383
+
return xerrors.Errorf("unmarshaling t.Ref: %w", err)
384
+
}
385
+
386
+
}
387
+
// t.Size (int64) (int64)
388
+
case "size":
389
+
{
390
+
maj, extra, err := cr.ReadHeader()
391
+
var extraI int64
392
+
if err != nil {
393
+
return err
394
+
}
395
+
switch maj {
396
+
case cbg.MajUnsignedInt:
397
+
extraI = int64(extra)
398
+
if extraI < 0 {
399
+
return fmt.Errorf("int64 positive overflow")
400
+
}
401
+
case cbg.MajNegativeInt:
402
+
extraI = int64(extra)
403
+
if extraI < 0 {
404
+
return fmt.Errorf("int64 negative overflow")
405
+
}
406
+
extraI = -1 - extraI
407
+
default:
408
+
return fmt.Errorf("wrong type for int64 field: %d", maj)
409
+
}
410
+
411
+
t.Size = int64(extraI)
412
+
}
413
+
// t.LexiconTypeID (string) (string)
414
+
case "$type":
415
+
416
+
{
417
+
sval, err := cbg.ReadString(cr)
418
+
if err != nil {
419
+
return err
420
+
}
421
+
422
+
t.LexiconTypeID = string(sval)
423
+
}
424
+
// t.MimeType (string) (string)
425
+
case "mimeType":
426
+
427
+
{
428
+
sval, err := cbg.ReadString(cr)
429
+
if err != nil {
430
+
return err
431
+
}
432
+
433
+
t.MimeType = string(sval)
434
+
}
435
+
436
+
default:
437
+
// Field doesn't exist on this type, so ignore it
438
+
cbg.ScanForLinks(r, func(cid.Cid) {})
439
+
}
440
+
}
441
+
442
+
return nil
443
+
}
+88
atproto/data/cidlink.go
+88
atproto/data/cidlink.go
···
1
+
package data
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
"io"
7
+
8
+
"github.com/ipfs/go-cid"
9
+
cbg "github.com/whyrusleeping/cbor-gen"
10
+
)
11
+
12
+
// Represents the "cid-link" type from the atproto data model.
13
+
//
14
+
// Implementation is a simple wrapper around the github.com/ipfs/go-cid "cid.Cid" type.
15
+
type CIDLink cid.Cid
16
+
17
+
type jsonLink struct {
18
+
Link string `json:"$link"`
19
+
}
20
+
21
+
// Unwraps the inner cid.Cid type (github.com/ipfs/go-cid)
22
+
func (ll CIDLink) CID() cid.Cid {
23
+
return cid.Cid(ll)
24
+
}
25
+
26
+
// Returns string representation.
27
+
//
28
+
// If the CID is "undefined", returns an empty string (note that this is different from how cid.Cid works).
29
+
func (ll CIDLink) String() string {
30
+
if ll.IsDefined() {
31
+
return cid.Cid(ll).String()
32
+
}
33
+
return ""
34
+
}
35
+
36
+
// Convenience helper, returns false if CID is "undefined" (golang zero value)
37
+
func (ll CIDLink) IsDefined() bool {
38
+
return cid.Cid(ll).Defined()
39
+
}
40
+
41
+
func (ll CIDLink) MarshalJSON() ([]byte, error) {
42
+
if !ll.IsDefined() {
43
+
return nil, fmt.Errorf("tried to marshal nil or undefined cid-link")
44
+
}
45
+
jl := jsonLink{
46
+
Link: ll.String(),
47
+
}
48
+
return json.Marshal(jl)
49
+
}
50
+
51
+
func (ll *CIDLink) UnmarshalJSON(raw []byte) error {
52
+
var jl jsonLink
53
+
if err := json.Unmarshal(raw, &jl); err != nil {
54
+
return fmt.Errorf("parsing cid-link JSON: %v", err)
55
+
}
56
+
57
+
c, err := cid.Decode(jl.Link)
58
+
if err != nil {
59
+
return fmt.Errorf("parsing cid-link CID: %v", err)
60
+
}
61
+
*ll = CIDLink(c)
62
+
return nil
63
+
}
64
+
65
+
func (ll *CIDLink) MarshalCBOR(w io.Writer) error {
66
+
if ll == nil {
67
+
_, err := w.Write(cbg.CborNull)
68
+
return err
69
+
}
70
+
if !ll.IsDefined() {
71
+
return fmt.Errorf("tried to marshal nil or undefined cid-link")
72
+
}
73
+
cw := cbg.NewCborWriter(w)
74
+
if err := cbg.WriteCid(cw, cid.Cid(*ll)); err != nil {
75
+
return fmt.Errorf("failed to write cid-link as CBOR: %w", err)
76
+
}
77
+
return nil
78
+
}
79
+
80
+
func (ll *CIDLink) UnmarshalCBOR(r io.Reader) error {
81
+
cr := cbg.NewCborReader(r)
82
+
c, err := cbg.ReadCid(cr)
83
+
if err != nil {
84
+
return fmt.Errorf("failed to read cid-link from CBOR: %w", err)
85
+
}
86
+
*ll = CIDLink(c)
87
+
return nil
88
+
}
+30
atproto/data/const.go
+30
atproto/data/const.go
···
1
+
package data
2
+
3
+
const (
4
+
// maximum size of any CBOR data, in any context, in atproto
5
+
MAX_CBOR_SIZE = 5 * 1024 * 1024
6
+
// maximum serialized size of an individual atproto record, in CBOR format
7
+
MAX_CBOR_RECORD_SIZE = 1 * 1024 * 1024
8
+
// maximum serialized size of an individual atproto record, in JSON format
9
+
MAX_JSON_RECORD_SIZE = 2 * 1024 * 1024
10
+
// maximum serialized size of blocks (raw bytes) in an atproto repo stream event (NOT ENFORCED YET)
11
+
MAX_STREAM_REPO_DIFF_SIZE = 4 * 1024 * 1024
12
+
// maximum size of a WebSocket frame in atproto event streams (NOT ENFORCED YET)
13
+
MAX_STREAM_FRAME_SIZE = MAX_CBOR_SIZE
14
+
// maximum size of any individual string inside an atproto record
15
+
MAX_RECORD_STRING_LEN = MAX_CBOR_RECORD_SIZE
16
+
// maximum size of any individual byte array (bytestring) inside an atproto record
17
+
MAX_RECORD_BYTES_LEN = MAX_CBOR_RECORD_SIZE
18
+
// limit on size of CID representation (NOT ENFORCED YET)
19
+
MAX_CID_BYTES = 100
20
+
// limit on depth of nested containers (objects or arrays) for atproto data (NOT ENFORCED YET)
21
+
MAX_CBOR_NESTED_LEVELS = 32
22
+
// maximum number of elements in an object or array in atproto data
23
+
MAX_CBOR_CONTAINER_LEN = 128 * 1024
24
+
// largest integer which can be represented in a float64. integers in atproto "should" not be larger than this. (NOT ENFORCED)
25
+
MAX_SAFE_INTEGER = 9007199254740991
26
+
// largest negative integer which can be represented in a float64. integers in atproto "should" not go below this. (NOT ENFORCED)
27
+
MIN_SAFE_INTEGER = -9007199254740991
28
+
// maximum length of string (UTF-8 bytes) in an atproto object (map)
29
+
MAX_OBJECT_KEY_LEN = 8192
30
+
)
+155
atproto/data/data.go
+155
atproto/data/data.go
···
1
+
package data
2
+
3
+
import (
4
+
"encoding/json"
5
+
"fmt"
6
+
7
+
"github.com/bluesky-social/indigo/atproto/syntax"
8
+
9
+
"github.com/ipfs/go-cid"
10
+
cbor "github.com/ipfs/go-ipld-cbor"
11
+
)
12
+
13
+
// Checks that generic data (object) complies with the atproto data model.
14
+
func Validate(obj map[string]any) error {
15
+
_, err := parseObject(obj)
16
+
return err
17
+
}
18
+
19
+
// Parses generic data (object) in JSON, validating against the atproto data model at the same time.
20
+
//
21
+
// The standard library's MarshalJSON can be used to invert this function.
22
+
func UnmarshalJSON(b []byte) (map[string]any, error) {
23
+
if len(b) > MAX_JSON_RECORD_SIZE {
24
+
return nil, fmt.Errorf("exceeded max JSON record size: %d", len(b))
25
+
}
26
+
var rawObj map[string]any
27
+
err := json.Unmarshal(b, &rawObj)
28
+
if err != nil {
29
+
return nil, err
30
+
}
31
+
out, err := parseObject(rawObj)
32
+
if err != nil {
33
+
return nil, err
34
+
}
35
+
return out, nil
36
+
}
37
+
38
+
// Parses generic data (object) in CBOR (specifically, IPLD dag-cbor), validating against the atproto data model at the same time.
39
+
func UnmarshalCBOR(b []byte) (map[string]any, error) {
40
+
if len(b) > MAX_CBOR_RECORD_SIZE {
41
+
return nil, fmt.Errorf("exceeded max CBOR record size: %d", len(b))
42
+
}
43
+
var rawObj map[string]any
44
+
err := cbor.DecodeInto(b, &rawObj)
45
+
if err != nil {
46
+
return nil, err
47
+
}
48
+
out, err := parseObject(rawObj)
49
+
if err != nil {
50
+
return nil, err
51
+
}
52
+
return out, nil
53
+
}
54
+
55
+
// Recursively finds all the "blob" objects from generic atproto data (which has already been parsed).
56
+
//
57
+
// Returns an array with all Blob instances; does not de-dupe.
58
+
func ExtractBlobs(obj map[string]any) []Blob {
59
+
return extractBlobsAtom(obj)
60
+
}
61
+
62
+
func extractBlobsAtom(atom any) []Blob {
63
+
out := []Blob{}
64
+
switch v := atom.(type) {
65
+
case Blob:
66
+
out = append(out, v)
67
+
case []any:
68
+
for _, el := range v {
69
+
down := extractBlobsAtom(el)
70
+
for _, d := range down {
71
+
out = append(out, d)
72
+
}
73
+
}
74
+
case map[string]any:
75
+
for _, val := range v {
76
+
down := extractBlobsAtom(val)
77
+
for _, d := range down {
78
+
out = append(out, d)
79
+
}
80
+
}
81
+
default:
82
+
}
83
+
return out
84
+
}
85
+
86
+
// Serializes generic atproto data (object) to DAG-CBOR bytes
87
+
//
88
+
// Does not re-validate that data conforms to atproto data model, but does handle Blob, Bytes, and CIDLink as expected.
89
+
func MarshalCBOR(obj map[string]any) ([]byte, error) {
90
+
return cbor.DumpObject(forCBOR(obj))
91
+
}
92
+
93
+
// helper to get generic data in the correct "shape" for serialization with ipfs/go-ipld-cbor
94
+
func forCBOR(obj map[string]any) map[string]any {
95
+
// NOTE: a faster version might mutate the map in-place instead of copying (many allocations)?
96
+
out := make(map[string]any, len(obj))
97
+
for k, val := range obj {
98
+
switch v := val.(type) {
99
+
case CIDLink:
100
+
out[k] = cid.Cid(v)
101
+
case Bytes:
102
+
out[k] = []byte(v)
103
+
case Blob:
104
+
out[k] = map[string]interface{}{
105
+
"$type": "blob",
106
+
"mimeType": v.MimeType,
107
+
"ref": cid.Cid(v.Ref),
108
+
"size": v.Size,
109
+
}
110
+
case syntax.AtIdentifier:
111
+
out[k] = v.String()
112
+
case *syntax.AtIdentifier:
113
+
out[k] = v.String()
114
+
case map[string]any:
115
+
out[k] = forCBOR(v)
116
+
case []any:
117
+
out[k] = forCBORArray(v)
118
+
default:
119
+
out[k] = v
120
+
}
121
+
}
122
+
return out
123
+
}
124
+
125
+
// recursive helper for forCBOR
126
+
func forCBORArray(arr []any) []any {
127
+
// NOTE: a faster version might mutate the array in-place instead of copying (many allocations)?
128
+
out := make([]any, len(arr))
129
+
for i, val := range arr {
130
+
switch v := val.(type) {
131
+
case CIDLink:
132
+
out[i] = cid.Cid(v)
133
+
case Bytes:
134
+
out[i] = []byte(v)
135
+
case Blob:
136
+
out[i] = map[string]interface{}{
137
+
"$type": "blob",
138
+
"mimeType": v.MimeType,
139
+
"ref": cid.Cid(v.Ref),
140
+
"size": v.Size,
141
+
}
142
+
case syntax.AtIdentifier:
143
+
out[i] = v.String()
144
+
case *syntax.AtIdentifier:
145
+
out[i] = v.String()
146
+
case map[string]any:
147
+
out[i] = forCBOR(v)
148
+
case []any:
149
+
out[i] = forCBORArray(v)
150
+
default:
151
+
out[i] = v
152
+
}
153
+
}
154
+
return out
155
+
}
+18
atproto/data/doc.go
+18
atproto/data/doc.go
···
1
+
/*
2
+
Package data supports schema-less serializaiton and deserialization of atproto data
3
+
4
+
Some restrictions from the data model include:
5
+
- string sizes
6
+
- array and object element counts
7
+
- the "shape" of $bytes and $blob data objects
8
+
- $type must contain a non-empty string
9
+
10
+
Details are specified at https://atproto.com/specs/data-model
11
+
12
+
This package includes types (CIDLink, Bytes, Blob) which are represent the corresponding atproto data model types. These implement JSON and CBOR marshaling in (with whyrusleeping/cbor-gen) the expected way.
13
+
14
+
Can parse generic atproto records (or other objects) in JSON or CBOR format in to map[string]interface{}, while validating atproto-specific constraints on data (eg, that cid-link objects have only a single field).
15
+
16
+
Has a helper for serializing generic data (map[string]interface{}) to CBOR, which handles converting JSON-style object types (like $link and $bytes) as needed. There is no "MarshalJSON" method; simply use the standard library's `encoding/json`.
17
+
*/
18
+
package data
+48
atproto/data/extract.go
+48
atproto/data/extract.go
···
1
+
package data
2
+
3
+
import (
4
+
"bytes"
5
+
"encoding/json"
6
+
"fmt"
7
+
"io"
8
+
)
9
+
10
+
// Helper type for extracting record $type from CBOR
11
+
type GenericRecord struct {
12
+
Type string `json:"$type" cborgen:"$type"`
13
+
}
14
+
15
+
// Parses the top-level $type field from generic atproto JSON data
16
+
func ExtractTypeJSON(b []byte) (string, error) {
17
+
var gr GenericRecord
18
+
if err := json.Unmarshal(b, &gr); err != nil {
19
+
return "", err
20
+
}
21
+
22
+
return gr.Type, nil
23
+
}
24
+
25
+
// Parses the top-level $type field from generic atproto CBOR data
26
+
func ExtractTypeCBOR(b []byte) (string, error) {
27
+
var gr GenericRecord
28
+
if err := gr.UnmarshalCBOR(bytes.NewReader(b)); err != nil {
29
+
fmt.Printf("bad bytes: %x\n", b)
30
+
return "", err
31
+
}
32
+
33
+
return gr.Type, nil
34
+
}
35
+
36
+
// Parses top-level $type field from generic atproto CBOR.
37
+
//
38
+
// Returns that string field, and additional bytes (TODO: the parsed bytes, or remaining bytes?)
39
+
func ExtractTypeCBORReader(r io.Reader) (string, []byte, error) {
40
+
buf := new(bytes.Buffer)
41
+
tr := io.TeeReader(r, buf)
42
+
var gr GenericRecord
43
+
if err := gr.UnmarshalCBOR(tr); err != nil {
44
+
return "", nil, err
45
+
}
46
+
47
+
return gr.Type, buf.Bytes(), nil
48
+
}
+41
atproto/data/extract_test.go
+41
atproto/data/extract_test.go
···
1
+
package data
2
+
3
+
import (
4
+
"io"
5
+
"os"
6
+
"testing"
7
+
8
+
"github.com/stretchr/testify/assert"
9
+
)
10
+
11
+
func TestExtract(t *testing.T) {
12
+
assert := assert.New(t)
13
+
14
+
// TODO: should this be an error?
15
+
tp, err := ExtractTypeJSON([]byte(`{
16
+
"type": "com.example.blah",
17
+
"a": 5
18
+
}`))
19
+
assert.NoError(err)
20
+
assert.Equal("", tp)
21
+
22
+
tp, err = ExtractTypeJSON([]byte(`{
23
+
"$type": "com.example.blah",
24
+
"a": 5
25
+
}`))
26
+
assert.NoError(err)
27
+
assert.Equal("com.example.blah", tp)
28
+
29
+
inFile, err := os.Open("testdata/feedpost_record.cbor")
30
+
if err != nil {
31
+
t.Fail()
32
+
}
33
+
cborBytes, err := io.ReadAll(inFile)
34
+
if err != nil {
35
+
t.Fail()
36
+
}
37
+
38
+
tp, err = ExtractTypeCBOR(cborBytes)
39
+
assert.NoError(err)
40
+
assert.Equal("app.bsky.feed.post", tp)
41
+
}
+134
atproto/data/interop_test.go
+134
atproto/data/interop_test.go
···
1
+
package data
2
+
3
+
import (
4
+
"encoding/base64"
5
+
"encoding/json"
6
+
"io"
7
+
"os"
8
+
"testing"
9
+
10
+
"github.com/ipfs/go-cid"
11
+
"github.com/stretchr/testify/assert"
12
+
)
13
+
14
+
type DataModelFixture struct {
15
+
JSON json.RawMessage `json:"json"`
16
+
CBORBase64 string `json:"cbor_base64"`
17
+
CID string `json:"cid"`
18
+
}
19
+
20
+
func TestInteropDataModelFixtures(t *testing.T) {
21
+
22
+
f, err := os.Open("testdata/data-model-fixtures.json")
23
+
if err != nil {
24
+
t.Fatal(err)
25
+
}
26
+
defer func() { _ = f.Close() }()
27
+
28
+
fixBytes, err := io.ReadAll(f)
29
+
if err != nil {
30
+
t.Fatal(err)
31
+
}
32
+
33
+
var fixtures []DataModelFixture
34
+
if err := json.Unmarshal(fixBytes, &fixtures); err != nil {
35
+
t.Fatal(err)
36
+
}
37
+
38
+
for _, row := range fixtures {
39
+
testDataModelFixture(t, row)
40
+
}
41
+
}
42
+
43
+
func testDataModelFixture(t *testing.T, row DataModelFixture) {
44
+
assert := assert.New(t)
45
+
46
+
jsonBytes := []byte(row.JSON)
47
+
cborBytes, err := base64.RawStdEncoding.DecodeString(row.CBORBase64)
48
+
if err != nil {
49
+
t.Fatal(err)
50
+
}
51
+
52
+
jsonObj, err := UnmarshalJSON(jsonBytes)
53
+
assert.NoError(err)
54
+
cborObj, err := UnmarshalCBOR(cborBytes)
55
+
assert.NoError(err)
56
+
57
+
assert.Equal(jsonObj, cborObj)
58
+
59
+
cborFromJSON, err := MarshalCBOR(jsonObj)
60
+
assert.NoError(err)
61
+
cborFromCBOR, err := MarshalCBOR(cborObj)
62
+
assert.NoError(err)
63
+
64
+
cborObjAgain, err := UnmarshalCBOR(cborFromJSON)
65
+
assert.NoError(err)
66
+
assert.Equal(jsonObj, cborObjAgain)
67
+
68
+
assert.Equal(cborBytes, cborFromJSON)
69
+
assert.Equal(cborBytes, cborFromCBOR)
70
+
71
+
// 0x71 = dag-cbor, 0x12 = sha2-256, 0 = default length
72
+
cidBuilder := cid.V1Builder{Codec: 0x71, MhType: 0x12, MhLength: 0}
73
+
cidFromJSON, err := cidBuilder.Sum(cborFromJSON)
74
+
assert.NoError(err)
75
+
assert.Equal(row.CID, cidFromJSON.String())
76
+
cidFromCBOR, err := cidBuilder.Sum(cborFromCBOR)
77
+
assert.NoError(err)
78
+
assert.Equal(row.CID, cidFromCBOR.String())
79
+
80
+
}
81
+
82
+
type DataModelSimpleFixture struct {
83
+
JSON json.RawMessage `json:"json"`
84
+
}
85
+
86
+
func TestInteropDataModelValid(t *testing.T) {
87
+
assert := assert.New(t)
88
+
89
+
f, err := os.Open("testdata/data-model-valid.json")
90
+
if err != nil {
91
+
t.Fatal(err)
92
+
}
93
+
defer func() { _ = f.Close() }()
94
+
95
+
fixBytes, err := io.ReadAll(f)
96
+
if err != nil {
97
+
t.Fatal(err)
98
+
}
99
+
100
+
var fixtures []DataModelSimpleFixture
101
+
if err := json.Unmarshal(fixBytes, &fixtures); err != nil {
102
+
t.Fatal(err)
103
+
}
104
+
105
+
for _, row := range fixtures {
106
+
_, err := UnmarshalJSON(row.JSON)
107
+
assert.NoError(err)
108
+
}
109
+
}
110
+
111
+
func TestInteropDataModelInvalid(t *testing.T) {
112
+
assert := assert.New(t)
113
+
114
+
f, err := os.Open("testdata/data-model-invalid.json")
115
+
if err != nil {
116
+
t.Fatal(err)
117
+
}
118
+
defer func() { _ = f.Close() }()
119
+
120
+
fixBytes, err := io.ReadAll(f)
121
+
if err != nil {
122
+
t.Fatal(err)
123
+
}
124
+
125
+
var fixtures []DataModelSimpleFixture
126
+
if err := json.Unmarshal(fixBytes, &fixtures); err != nil {
127
+
t.Fatal(err)
128
+
}
129
+
130
+
for _, row := range fixtures {
131
+
_, err := UnmarshalJSON(row.JSON)
132
+
assert.Error(err)
133
+
}
134
+
}
+225
atproto/data/parse.go
+225
atproto/data/parse.go
···
1
+
package data
2
+
3
+
import (
4
+
"encoding"
5
+
"encoding/base64"
6
+
"fmt"
7
+
"reflect"
8
+
9
+
"github.com/ipfs/go-cid"
10
+
)
11
+
12
+
func parseFloat(f float64) (int64, error) {
13
+
if f != float64(int64(f)) {
14
+
return 0, fmt.Errorf("number was is not a safe integer: %f", f)
15
+
}
16
+
return int64(f), nil
17
+
}
18
+
19
+
func parseAtom(atom any) (any, error) {
20
+
switch v := atom.(type) {
21
+
case nil:
22
+
return v, nil
23
+
case bool:
24
+
return v, nil
25
+
case *bool:
26
+
return *v, nil
27
+
case int64:
28
+
return v, nil
29
+
case *int64:
30
+
return *v, nil
31
+
case int:
32
+
return int64(v), nil
33
+
case *int:
34
+
return int64(*v), nil
35
+
case float64:
36
+
return parseFloat(v)
37
+
case *float64:
38
+
return parseFloat(*v)
39
+
case string:
40
+
if len(v) > MAX_RECORD_STRING_LEN {
41
+
return nil, fmt.Errorf("string too long: %d", len(v))
42
+
}
43
+
return v, nil
44
+
case *string:
45
+
return parseAtom(*v)
46
+
case cid.Cid:
47
+
return CIDLink(v), nil
48
+
case *cid.Cid:
49
+
return CIDLink(*v), nil
50
+
case []byte:
51
+
return Bytes(v), nil
52
+
case *[]byte:
53
+
return Bytes(*v), nil
54
+
case []any:
55
+
return parseArray(v)
56
+
case *[]any:
57
+
return parseArray(*v)
58
+
case map[string]any:
59
+
return parseMap(v)
60
+
case *map[string]any:
61
+
return parseMap(*v)
62
+
case encoding.TextMarshaler:
63
+
s, err := v.MarshalText()
64
+
if err != nil {
65
+
return nil, fmt.Errorf("failed to marshal text (%s): %w", reflect.TypeOf(v), err)
66
+
}
67
+
return s, nil
68
+
default:
69
+
return nil, fmt.Errorf("unexpected type: %s", reflect.TypeOf(v))
70
+
}
71
+
}
72
+
73
+
func parseArray(l []any) ([]any, error) {
74
+
if len(l) > MAX_CBOR_CONTAINER_LEN {
75
+
return nil, fmt.Errorf("data array length too long: %d", len(l))
76
+
}
77
+
out := make([]any, len(l))
78
+
for i, v := range l {
79
+
p, err := parseAtom(v)
80
+
if err != nil {
81
+
return nil, err
82
+
}
83
+
out[i] = p
84
+
}
85
+
return out, nil
86
+
}
87
+
88
+
func parseMap(obj map[string]any) (any, error) {
89
+
if len(obj) > MAX_CBOR_CONTAINER_LEN {
90
+
return nil, fmt.Errorf("data object has too many fields: %d", len(obj))
91
+
}
92
+
if _, ok := obj["$link"]; ok {
93
+
return parseLink(obj)
94
+
}
95
+
if _, ok := obj["$bytes"]; ok {
96
+
return parseBytes(obj)
97
+
}
98
+
if typeVal, ok := obj["$type"]; ok {
99
+
if typeStr, ok := typeVal.(string); ok {
100
+
if typeStr == "blob" {
101
+
b, err := parseBlob(obj)
102
+
if err != nil {
103
+
return nil, err
104
+
}
105
+
return *b, nil
106
+
}
107
+
if len(typeStr) == 0 {
108
+
return nil, fmt.Errorf("$type field must contain a non-empty string")
109
+
}
110
+
} else {
111
+
return nil, fmt.Errorf("$type field must contain a non-empty string")
112
+
}
113
+
}
114
+
out := make(map[string]any, len(obj))
115
+
for k, val := range obj {
116
+
if len(k) > MAX_OBJECT_KEY_LEN {
117
+
return nil, fmt.Errorf("data object key too long: %d", len(k))
118
+
}
119
+
atom, err := parseAtom(val)
120
+
if err != nil {
121
+
return nil, err
122
+
}
123
+
out[k] = atom
124
+
}
125
+
return out, nil
126
+
}
127
+
128
+
func parseLink(obj map[string]any) (CIDLink, error) {
129
+
var zero CIDLink
130
+
if len(obj) != 1 {
131
+
return zero, fmt.Errorf("$link objects must have a single field")
132
+
}
133
+
v, ok := obj["$link"].(string)
134
+
if !ok {
135
+
return zero, fmt.Errorf("$link field missing or not a string")
136
+
}
137
+
c, err := cid.Parse(v)
138
+
if err != nil {
139
+
return zero, fmt.Errorf("invalid $link CID: %w", err)
140
+
}
141
+
if !c.Defined() {
142
+
return zero, fmt.Errorf("undefined (null) CID in $link")
143
+
}
144
+
return CIDLink(c), nil
145
+
}
146
+
147
+
func parseBytes(obj map[string]any) (Bytes, error) {
148
+
if len(obj) != 1 {
149
+
return nil, fmt.Errorf("$bytes objects must have a single field")
150
+
}
151
+
v, ok := obj["$bytes"].(string)
152
+
if !ok {
153
+
return nil, fmt.Errorf("$bytes field missing or not a string")
154
+
}
155
+
b, err := base64.RawStdEncoding.DecodeString(v)
156
+
if err != nil {
157
+
return nil, fmt.Errorf("decoding $byte value: %w", err)
158
+
}
159
+
return Bytes(b), nil
160
+
}
161
+
162
+
// NOTE: doesn't handle legacy blobs yet!
163
+
func parseBlob(obj map[string]any) (*Blob, error) {
164
+
if len(obj) != 4 {
165
+
return nil, fmt.Errorf("blobs expected to have 4 fields")
166
+
}
167
+
if obj["$type"] != "blob" {
168
+
return nil, fmt.Errorf("blobs expected to have $type=blob")
169
+
}
170
+
var size int64
171
+
var err error
172
+
switch v := obj["size"].(type) {
173
+
case int:
174
+
size = int64(v)
175
+
case int64:
176
+
size = v
177
+
case float64:
178
+
size, err = parseFloat(v)
179
+
if err != nil {
180
+
return nil, err
181
+
}
182
+
default:
183
+
return nil, fmt.Errorf("blob 'size' missing or not a number")
184
+
}
185
+
mimeType, ok := obj["mimeType"].(string)
186
+
if !ok {
187
+
return nil, fmt.Errorf("blob 'mimeType' missing or not a string")
188
+
}
189
+
rawRef, ok := obj["ref"]
190
+
if !ok {
191
+
return nil, fmt.Errorf("blob 'ref' missing")
192
+
}
193
+
var ref CIDLink
194
+
switch v := rawRef.(type) {
195
+
case map[string]any:
196
+
cl, err := parseLink(v)
197
+
if err != nil {
198
+
return nil, err
199
+
}
200
+
ref = cl
201
+
case cid.Cid:
202
+
ref = CIDLink(v)
203
+
case CIDLink:
204
+
ref = v
205
+
default:
206
+
return nil, fmt.Errorf("blob 'ref' unexpected type")
207
+
}
208
+
209
+
return &Blob{
210
+
Size: size,
211
+
MimeType: mimeType,
212
+
Ref: ref,
213
+
}, nil
214
+
}
215
+
216
+
func parseObject(obj map[string]any) (map[string]any, error) {
217
+
out, err := parseMap(obj)
218
+
if err != nil {
219
+
return nil, err
220
+
}
221
+
if outObj, ok := out.(map[string]any); ok {
222
+
return outObj, nil
223
+
}
224
+
return nil, fmt.Errorf("top-level datum was not an object")
225
+
}
+60
atproto/data/testdata/data-model-fixtures.json
+60
atproto/data/testdata/data-model-fixtures.json
···
1
+
[
2
+
{
3
+
"json": {
4
+
"string": "abc",
5
+
"unicode": "a~öñ©⽘☎𓋓😀👨👩👧👧",
6
+
"integer": 123,
7
+
"bool": true,
8
+
"null": null,
9
+
"array": ["abc", "def", "ghi"],
10
+
"object": {
11
+
"string": "abc",
12
+
"number": 123,
13
+
"bool": true,
14
+
"arr": ["abc", "def", "ghi"]
15
+
}
16
+
},
17
+
"cbor_base64": "p2Rib29s9WRudWxs9mVhcnJheYNjYWJjY2RlZmNnaGlmb2JqZWN0pGNhcnKDY2FiY2NkZWZjZ2hpZGJvb2z1Zm51bWJlchh7ZnN0cmluZ2NhYmNmc3RyaW5nY2FiY2dpbnRlZ2VyGHtndW5pY29kZXgvYX7DtsOxwqnivZjimI7wk4uT8J+YgPCfkajigI3wn5Gp4oCN8J+Rp+KAjfCfkac",
18
+
"cid": "bafyreiclp443lavogvhj3d2ob2cxbfuscni2k5jk7bebjzg7khl3esabwq"
19
+
},
20
+
{
21
+
"json": {
22
+
"a": {
23
+
"$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"
24
+
},
25
+
"b": {
26
+
"$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0"
27
+
},
28
+
"c": {
29
+
"$type": "blob",
30
+
"ref": {
31
+
"$link": "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity"
32
+
},
33
+
"mimeType": "image/jpeg",
34
+
"size": 10000
35
+
}
36
+
},
37
+
"cbor_base64": "o2Fh2CpYJQABcRIgZQYqWloA/BbXPGlEI3zLwVscSnI0SJM2iR0JF0GiOdBhYlggnFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI1hY6RjcmVm2CpYJQABVRIgQljP/3j2E2l2l1Y/kmyR5dNXTS6iWuftktbr/COjiJ5kc2l6ZRknEGUkdHlwZWRibG9iaG1pbWVUeXBlamltYWdlL2pwZWc",
38
+
"cid": "bafyreihldkhcwijkde7gx4rpkkuw7pl6lbyu5gieunyc7ihactn5bkd2nm"
39
+
},
40
+
{
41
+
"json": {
42
+
"a": {
43
+
"b": [
44
+
{
45
+
"d": [
46
+
{"$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"},
47
+
{"$link": "bafyreidfayvfuwqa7qlnopdjiqrxzs6blmoeu4rujcjtnci5beludirz2a"}
48
+
],
49
+
"e": [
50
+
{ "$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0" },
51
+
{ "$bytes": "iE+sPoHobU9tSIqGI+309LLCcWQIRmEXwxcoDt19tas" }
52
+
]
53
+
}
54
+
]
55
+
}
56
+
},
57
+
"cbor_base64": "oWFhoWFigaJhZILYKlglAAFxEiBlBipaWgD8Ftc8aUQjfMvBWxxKcjRIkzaJHQkXQaI50NgqWCUAAXESIGUGKlpaAPwW1zxpRCN8y8FbHEpyNEiTNokdCRdBojnQYWWCWCCcURGO8suLD2qbjkmuof1BPPILYu7Vdvid7r6wGsLMjVggiE+sPoHobU9tSIqGI+309LLCcWQIRmEXwxcoDt19tas",
58
+
"cid": "bafyreid3imdulnhgeytpf6uk7zahjvrsqlofkmm5b5ub2maw4kqus6jp4i"
59
+
}
60
+
]
+111
atproto/data/testdata/data-model-invalid.json
+111
atproto/data/testdata/data-model-invalid.json
···
1
+
[
2
+
{
3
+
"note": "top-level not an object",
4
+
"json": "blah"
5
+
},
6
+
{
7
+
"note": "float",
8
+
"json": {
9
+
"rcrd": {
10
+
"$type": "com.example.blah",
11
+
"a": 123.456,
12
+
"b": "blah"
13
+
}
14
+
}
15
+
},
16
+
{
17
+
"note": "record with $type null",
18
+
"json": {
19
+
"rcrd": {
20
+
"$type": null,
21
+
"a": 123,
22
+
"b": "blah"
23
+
}
24
+
}
25
+
},
26
+
{
27
+
"note": "record with $type wrong type",
28
+
"json": {
29
+
"rcrd": {
30
+
"$type": 123,
31
+
"a": 123,
32
+
"b": "blah"
33
+
}
34
+
}
35
+
},
36
+
{
37
+
"note": "record with empty $type string",
38
+
"json": {
39
+
"rcrd": {
40
+
"$type": "",
41
+
"a": 123,
42
+
"b": "blah"
43
+
}
44
+
}
45
+
},
46
+
{
47
+
"note": "blob with string size",
48
+
"json": {
49
+
"blb": {
50
+
"$type": "blob",
51
+
"ref": {
52
+
"$link": "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity"
53
+
},
54
+
"mimeType": "image/jpeg",
55
+
"size": "10000"
56
+
}
57
+
}
58
+
},
59
+
{
60
+
"note": "blob with missing key",
61
+
"json": {
62
+
"blb": {
63
+
"$type": "blob",
64
+
"mimeType": "image/jpeg",
65
+
"size": 10000
66
+
}
67
+
}
68
+
},
69
+
{
70
+
"note": "bytes with wrong field type",
71
+
"json": {
72
+
"lnk": {
73
+
"$bytes": [1,2,3]
74
+
}
75
+
}
76
+
},
77
+
{
78
+
"note": "bytes with extra fields",
79
+
"json": {
80
+
"lnk": {
81
+
"$bytes": "nFERjvLLiw9qm45JrqH9QTzyC2Lu1Xb4ne6+sBrCzI0",
82
+
"other": "blah"
83
+
}
84
+
}
85
+
},
86
+
{
87
+
"note": "link with wrong field type",
88
+
"json": {
89
+
"lnk": {
90
+
"$link": 1234
91
+
}
92
+
}
93
+
},
94
+
{
95
+
"note": "link with bogus CID",
96
+
"json": {
97
+
"lnk": {
98
+
"$link": "."
99
+
}
100
+
}
101
+
},
102
+
{
103
+
"note": "link with extra fields",
104
+
"json": {
105
+
"lnk": {
106
+
"$link": "bafkreiccldh766hwcnuxnf2wh6jgzepf2nlu2lvcllt63eww5p6chi4ity",
107
+
"other": "blah"
108
+
}
109
+
}
110
+
}
111
+
]
+48
atproto/data/testdata/data-model-valid.json
+48
atproto/data/testdata/data-model-valid.json
···
1
+
[
2
+
{
3
+
"note": "trivial record",
4
+
"json": {
5
+
"rcrd": {
6
+
"$type": "com.example.blah",
7
+
"a": 123,
8
+
"b": "blah"
9
+
}
10
+
}
11
+
},
12
+
{
13
+
"note": "float, but integer-like",
14
+
"json": {
15
+
"rcrd": {
16
+
"$type": "com.example.blah",
17
+
"a": 123.0,
18
+
"b": "blah"
19
+
}
20
+
}
21
+
},
22
+
{
23
+
"note": "empty list and object",
24
+
"json": {
25
+
"rcrd": {
26
+
"$type": "com.example.blah",
27
+
"a": [],
28
+
"b": {}
29
+
}
30
+
}
31
+
},
32
+
{
33
+
"note": "list of nullable",
34
+
"json": {
35
+
"arr": [1,2,null]
36
+
}
37
+
},
38
+
{
39
+
"note": "list of lists",
40
+
"json": {
41
+
"arr": [
42
+
[1,2,3],
43
+
[4,5,6]
44
+
],
45
+
"arr2": [null, null, null]
46
+
}
47
+
}
48
+
]
atproto/data/testdata/feedpost_record.cbor
atproto/data/testdata/feedpost_record.cbor
This is a binary file and will not be displayed.
+6
gen/main.go
+6
gen/main.go
···
10
10
lexutil "github.com/bluesky-social/indigo/lex/util"
11
11
"github.com/bluesky-social/indigo/mst"
12
12
"github.com/bluesky-social/indigo/repo"
13
+
"github.com/bluesky-social/indigo/atproto/data"
14
+
13
15
cbg "github.com/whyrusleeping/cbor-gen"
14
16
)
15
17
···
83
85
}
84
86
85
87
if err := cbg.WriteMapEncodersToFile("events/cbor_gen.go", "events", events.EventHeader{}, events.ErrorFrame{}); err != nil {
88
+
panic(err)
89
+
}
90
+
91
+
if err := cbg.WriteMapEncodersToFile("atproto/data/cbor_gen.go", "data", data.GenericRecord{}, data.LegacyBlobSchema{}, data.BlobSchema{}); err != nil {
86
92
panic(err)
87
93
}
88
94
}