lex: refactor $blob, $link, and $bytes serialization · alice.mosphere.at/indigo@8a404c3

+1 -1

gen/main.go

··· 32 32 panic(err) 33 33 } 34 34 35 - if err := cbg.WriteMapEncodersToFile("lex/util/cbor_gen.go", "util", lexutil.CborChecker{}, lexutil.Blob{}); err != nil { 35 + if err := cbg.WriteMapEncodersToFile("lex/util/cbor_gen.go", "util", lexutil.CborChecker{}, lexutil.LegacyBlob{}, lexutil.BlobSchema{}); err != nil { 36 36 panic(err) 37 37 } 38 38

+4 -4

lex/gen.go

··· 1018 1018 return "util.LexiconTypeDecoder", nil 1019 1019 case "union": 1020 1020 return "*" + name + "_" + strings.Title(k), nil 1021 - case "image": 1022 - return "*util.Blob", nil 1023 1021 case "blob": 1024 - return "*util.Blob", nil 1022 + return "*util.LexBlob", nil 1025 1023 case "array": 1026 1024 subt, err := s.typeNameForField(name+"_"+strings.Title(k), "Elem", *v.Items) 1027 1025 if err != nil { ··· 1030 1028 1031 1029 return "[]" + subt, nil 1032 1030 case "cid-link": 1033 - return "cid.Cid", nil 1031 + return "util.LexLink", nil 1032 + case "bytes": 1033 + return "util.LexBytes", nil 1034 1034 default: 1035 1035 return "", fmt.Errorf("field %q in %s has unsupported type name (%s)", k, name, v.Type) 1036 1036 }

+174 -14

lex/util/cbor_gen.go

··· 113 113 114 114 return nil 115 115 } 116 - func (t *Blob) MarshalCBOR(w io.Writer) error { 116 + func (t *LegacyBlob) MarshalCBOR(w io.Writer) error { 117 117 if t == nil { 118 118 _, err := w.Write(cbg.CborNull) 119 119 return err ··· 121 121 122 122 cw := cbg.NewCborWriter(w) 123 123 124 - if _, err := cw.Write([]byte{163}); err != nil { 124 + if _, err := cw.Write([]byte{162}); err != nil { 125 + return err 126 + } 127 + 128 + // t.Cid (string) (string) 129 + if len("cid") > cbg.MaxLength { 130 + return xerrors.Errorf("Value in field \"cid\" was too long") 131 + } 132 + 133 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cid"))); err != nil { 134 + return err 135 + } 136 + if _, err := io.WriteString(w, string("cid")); err != nil { 137 + return err 138 + } 139 + 140 + if len(t.Cid) > cbg.MaxLength { 141 + return xerrors.Errorf("Value in field t.Cid was too long") 142 + } 143 + 144 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Cid))); err != nil { 145 + return err 146 + } 147 + if _, err := io.WriteString(w, string(t.Cid)); err != nil { 148 + return err 149 + } 150 + 151 + // t.MimeType (string) (string) 152 + if len("mimeType") > cbg.MaxLength { 153 + return xerrors.Errorf("Value in field \"mimeType\" was too long") 154 + } 155 + 156 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("mimeType"))); err != nil { 157 + return err 158 + } 159 + if _, err := io.WriteString(w, string("mimeType")); err != nil { 125 160 return err 126 161 } 127 162 128 - // t.Ref (cid.Cid) (struct) 163 + if len(t.MimeType) > cbg.MaxLength { 164 + return xerrors.Errorf("Value in field t.MimeType was too long") 165 + } 166 + 167 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.MimeType))); err != nil { 168 + return err 169 + } 170 + if _, err := io.WriteString(w, string(t.MimeType)); err != nil { 171 + return err 172 + } 173 + return nil 174 + } 175 + 176 + func (t *LegacyBlob) UnmarshalCBOR(r io.Reader) (err error) { 177 + *t = LegacyBlob{} 178 + 179 + cr := cbg.NewCborReader(r) 180 + 181 + maj, extra, err := cr.ReadHeader() 182 + if err != nil { 183 + return err 184 + } 185 + defer func() { 186 + if err == io.EOF { 187 + err = io.ErrUnexpectedEOF 188 + } 189 + }() 190 + 191 + if maj != cbg.MajMap { 192 + return fmt.Errorf("cbor input should be of type map") 193 + } 194 + 195 + if extra > cbg.MaxLength { 196 + return fmt.Errorf("LegacyBlob: map struct too large (%d)", extra) 197 + } 198 + 199 + var name string 200 + n := extra 201 + 202 + for i := uint64(0); i < n; i++ { 203 + 204 + { 205 + sval, err := cbg.ReadString(cr) 206 + if err != nil { 207 + return err 208 + } 209 + 210 + name = string(sval) 211 + } 212 + 213 + switch name { 214 + // t.Cid (string) (string) 215 + case "cid": 216 + 217 + { 218 + sval, err := cbg.ReadString(cr) 219 + if err != nil { 220 + return err 221 + } 222 + 223 + t.Cid = string(sval) 224 + } 225 + // t.MimeType (string) (string) 226 + case "mimeType": 227 + 228 + { 229 + sval, err := cbg.ReadString(cr) 230 + if err != nil { 231 + return err 232 + } 233 + 234 + t.MimeType = string(sval) 235 + } 236 + 237 + default: 238 + // Field doesn't exist on this type, so ignore it 239 + cbg.ScanForLinks(r, func(cid.Cid) {}) 240 + } 241 + } 242 + 243 + return nil 244 + } 245 + func (t *BlobSchema) MarshalCBOR(w io.Writer) error { 246 + if t == nil { 247 + _, err := w.Write(cbg.CborNull) 248 + return err 249 + } 250 + 251 + cw := cbg.NewCborWriter(w) 252 + 253 + if _, err := cw.Write([]byte{164}); err != nil { 254 + return err 255 + } 256 + 257 + // t.Ref (util.LexLink) (struct) 129 258 if len("ref") > cbg.MaxLength { 130 259 return xerrors.Errorf("Value in field \"ref\" was too long") 131 260 } ··· 137 266 return err 138 267 } 139 268 140 - if err := cbg.WriteCid(cw, t.Ref); err != nil { 141 - return xerrors.Errorf("failed to write cid field t.Ref: %w", err) 269 + if err := t.Ref.MarshalCBOR(cw); err != nil { 270 + return err 142 271 } 143 272 144 273 // t.Size (int64) (int64) ··· 185 314 if _, err := io.WriteString(w, string(t.MimeType)); err != nil { 186 315 return err 187 316 } 317 + 318 + // t.LexiconTypeID (string) (string) 319 + if len("LexiconTypeID") > cbg.MaxLength { 320 + return xerrors.Errorf("Value in field \"LexiconTypeID\" was too long") 321 + } 322 + 323 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("LexiconTypeID"))); err != nil { 324 + return err 325 + } 326 + if _, err := io.WriteString(w, string("LexiconTypeID")); err != nil { 327 + return err 328 + } 329 + 330 + if len(t.LexiconTypeID) > cbg.MaxLength { 331 + return xerrors.Errorf("Value in field t.LexiconTypeID was too long") 332 + } 333 + 334 + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.LexiconTypeID))); err != nil { 335 + return err 336 + } 337 + if _, err := io.WriteString(w, string(t.LexiconTypeID)); err != nil { 338 + return err 339 + } 188 340 return nil 189 341 } 190 342 191 - func (t *Blob) UnmarshalCBOR(r io.Reader) (err error) { 192 - *t = Blob{} 343 + func (t *BlobSchema) UnmarshalCBOR(r io.Reader) (err error) { 344 + *t = BlobSchema{} 193 345 194 346 cr := cbg.NewCborReader(r) 195 347 ··· 208 360 } 209 361 210 362 if extra > cbg.MaxLength { 211 - return fmt.Errorf("Blob: map struct too large (%d)", extra) 363 + return fmt.Errorf("BlobSchema: map struct too large (%d)", extra) 212 364 } 213 365 214 366 var name string ··· 226 378 } 227 379 228 380 switch name { 229 - // t.Ref (cid.Cid) (struct) 381 + // t.Ref (util.LexLink) (struct) 230 382 case "ref": 231 383 232 384 { 233 385 234 - c, err := cbg.ReadCid(cr) 235 - if err != nil { 236 - return xerrors.Errorf("failed to read cid field t.Ref: %w", err) 386 + if err := t.Ref.UnmarshalCBOR(cr); err != nil { 387 + return xerrors.Errorf("unmarshaling t.Ref: %w", err) 237 388 } 238 - 239 - t.Ref = c 240 389 241 390 } 242 391 // t.Size (int64) (int64) ··· 275 424 } 276 425 277 426 t.MimeType = string(sval) 427 + } 428 + // t.LexiconTypeID (string) (string) 429 + case "LexiconTypeID": 430 + 431 + { 432 + sval, err := cbg.ReadString(cr) 433 + if err != nil { 434 + return err 435 + } 436 + 437 + t.LexiconTypeID = string(sval) 278 438 } 279 439 280 440 default:

+262

lex/util/lex_types.go

··· 1 + package util 2 + 3 + import ( 4 + "bytes" 5 + "encoding/base64" 6 + "encoding/json" 7 + "io" 8 + 9 + "github.com/ipfs/go-cid" 10 + cbg "github.com/whyrusleeping/cbor-gen" 11 + xerrors "golang.org/x/xerrors" 12 + ) 13 + 14 + const ( 15 + // TODO: this is an arbitrary size. lexicons can set more realistic limits, 16 + // and we should pass those limits through and only fall back to this when 17 + // undefined. 18 + MAX_BYTE_ARRAY_SIZE = 128 * 1024 * 1024 19 + ) 20 + 21 + type LexLink cid.Cid 22 + 23 + type jsonLink struct { 24 + Link string `json:"$type"` 25 + } 26 + 27 + // convenience helper 28 + func (ll LexLink) String() string { 29 + return cid.Cid(ll).String() 30 + } 31 + 32 + // convenience helper 33 + func (ll LexLink) Defined() bool { 34 + return cid.Cid(ll).Defined() 35 + } 36 + 37 + func (ll *LexLink) MarshalJSON() ([]byte, error) { 38 + if ll == nil || !ll.Defined() { 39 + return nil, xerrors.Errorf("tried to marshal nil or undefined cid-link") 40 + } 41 + jl := jsonLink{ 42 + Link: (*cid.Cid)(ll).String(), 43 + } 44 + return json.Marshal(jl) 45 + } 46 + 47 + func (ll *LexLink) UnmarshalJSON(raw []byte) error { 48 + var jl jsonLink 49 + err := json.Unmarshal(raw, &jl) 50 + if err != nil { 51 + return xerrors.Errorf("parsing cid-link JSON: %v", err) 52 + } 53 + c, err := cid.Decode(jl.Link) 54 + if err != nil { 55 + return xerrors.Errorf("parsing cid-link CID: %v", err) 56 + } 57 + *ll = LexLink(c) 58 + return nil 59 + } 60 + 61 + func (ll *LexLink) MarshalCBOR(w io.Writer) error { 62 + if ll == nil || !ll.Defined() { 63 + return xerrors.Errorf("tried to marshal nil or undefined cid-link") 64 + } 65 + cw := cbg.NewCborWriter(w) 66 + if err := cbg.WriteCid(cw, cid.Cid(*ll)); err != nil { 67 + return xerrors.Errorf("failed to write cid-link as CBOR: %w", err) 68 + } 69 + return nil 70 + } 71 + 72 + func (ll *LexLink) UnmarshalCBOR(r io.Reader) error { 73 + cr := cbg.NewCborReader(r) 74 + c, err := cbg.ReadCid(cr) 75 + if err != nil { 76 + return xerrors.Errorf("failed to read cid-link from CBOR: %w", err) 77 + } 78 + *ll = LexLink(c) 79 + return nil 80 + } 81 + 82 + type LexBytes []byte 83 + 84 + type jsonBytes struct { 85 + Bytes string `json:"$bytes"` 86 + } 87 + 88 + func (lb *LexBytes) MarshalJSON() ([]byte, error) { 89 + if lb == nil { 90 + return nil, xerrors.Errorf("tried to marshal nil $bytes") 91 + } 92 + jb := jsonBytes{ 93 + Bytes: base64.StdEncoding.EncodeToString([]byte(*lb)), 94 + } 95 + return json.Marshal(jb) 96 + } 97 + 98 + func (lb *LexBytes) UnmarshalJSON(raw []byte) error { 99 + var jb jsonBytes 100 + err := json.Unmarshal(raw, &jb) 101 + if err != nil { 102 + return xerrors.Errorf("parsing $bytes JSON: %v", err) 103 + } 104 + out, err := base64.StdEncoding.DecodeString(jb.Bytes) 105 + if err != nil { 106 + return xerrors.Errorf("parsing $bytes base64: %v", err) 107 + } 108 + *lb = LexBytes(out) 109 + return nil 110 + } 111 + 112 + func (lb *LexBytes) MarshalCBOR(w io.Writer) error { 113 + if lb == nil { 114 + return xerrors.Errorf("tried to marshal nil or undefined $bytes") 115 + } 116 + cw := cbg.NewCborWriter(w) 117 + if err := cbg.WriteByteArray(cw, ([]byte)(*lb)); err != nil { 118 + return xerrors.Errorf("failed to write $bytes as CBOR: %w", err) 119 + } 120 + return nil 121 + } 122 + 123 + func (lb *LexBytes) UnmarshalCBOR(r io.Reader) error { 124 + cr := cbg.NewCborReader(r) 125 + b, err := cbg.ReadByteArray(cr, MAX_BYTE_ARRAY_SIZE) 126 + if err != nil { 127 + return xerrors.Errorf("failed to read $bytes from CBOR: %w", err) 128 + } 129 + *lb = LexBytes(b) 130 + return nil 131 + } 132 + 133 + // used in schemas, and can represent either a legacy blob or a "new" (lex 134 + // refactor) blob. size=-1 indicates that this is (and should be serialized as) 135 + // a legacy blob (string CID, no size, etc). 136 + type LexBlob struct { 137 + Ref LexLink `json:"ref" cborgen:"ref"` 138 + MimeType string `json:"mimeType" cborgen:"mimeType"` 139 + Size int64 `json:"size" cborgen:"size"` 140 + } 141 + 142 + type LegacyBlob struct { 143 + Cid string `json:"cid" cborgen:"cid"` 144 + MimeType string `json:"mimeType" cborgen:"mimeType"` 145 + } 146 + 147 + type BlobSchema struct { 148 + LexiconTypeID string `json:"$type,omitempty"` 149 + Ref LexLink `json:"ref" cborgen:"ref"` 150 + MimeType string `json:"mimeType" cborgen:"mimeType"` 151 + Size int64 `json:"size" cborgen:"size"` 152 + } 153 + 154 + func (b *LexBlob) MarshalJSON() ([]byte, error) { 155 + if b.Size < 0 { 156 + lb := LegacyBlob{ 157 + Cid: b.Ref.String(), 158 + MimeType: b.MimeType, 159 + } 160 + return json.Marshal(lb) 161 + } else { 162 + nb := BlobSchema{ 163 + LexiconTypeID: "blob", 164 + Ref: b.Ref, 165 + MimeType: b.MimeType, 166 + Size: b.Size, 167 + } 168 + return json.Marshal(nb) 169 + } 170 + } 171 + 172 + func (b *LexBlob) UnmarshalJSON(raw []byte) error { 173 + typ, err := TypeExtract(raw) 174 + if err != nil { 175 + return xerrors.Errorf("parsing blob: %v", err) 176 + } 177 + 178 + if typ == "blob" { 179 + var bs BlobSchema 180 + err := json.Unmarshal(raw, &bs) 181 + if err != nil { 182 + return xerrors.Errorf("parsing blob JSON: %v", err) 183 + } 184 + b.Ref = bs.Ref 185 + b.MimeType = bs.MimeType 186 + b.Size = bs.Size 187 + if bs.Size < 0 { 188 + return xerrors.Errorf("parsing blob: negative size: %d", bs.Size) 189 + } 190 + } else { 191 + var legacy *LegacyBlob 192 + err := json.Unmarshal(raw, legacy) 193 + if err != nil { 194 + return xerrors.Errorf("parsing legacy blob: %v", err) 195 + } 196 + refCid, err := cid.Decode(legacy.Cid) 197 + if err != nil { 198 + return xerrors.Errorf("parsing CID in legacy blob: %v", err) 199 + } 200 + b.Ref = LexLink(refCid) 201 + b.MimeType = legacy.MimeType 202 + b.Size = -1 203 + } 204 + return nil 205 + } 206 + 207 + func (b *LexBlob) MarshalCBOR(w io.Writer) error { 208 + if b == nil { 209 + return nil 210 + } 211 + if b.Size < 0 { 212 + lb := LegacyBlob{ 213 + Cid: b.Ref.String(), 214 + MimeType: b.MimeType, 215 + } 216 + return lb.MarshalCBOR(w) 217 + } else { 218 + bs := BlobSchema{ 219 + LexiconTypeID: "blob", 220 + Ref: b.Ref, 221 + MimeType: b.MimeType, 222 + Size: b.Size, 223 + } 224 + return bs.MarshalCBOR(w) 225 + } 226 + } 227 + 228 + func (lb *LexBlob) UnmarshalCBOR(r io.Reader) error { 229 + typ, b, err := CborTypeExtractReader(r) 230 + if err != nil { 231 + return xerrors.Errorf("parsing $blob CBOR type: %w", err) 232 + } 233 + *lb = LexBlob{} 234 + if typ == "blob" { 235 + var bs BlobSchema 236 + err := bs.UnmarshalCBOR(bytes.NewReader(b)) 237 + if err != nil { 238 + return xerrors.Errorf("parsing $blob CBOR: %v", err) 239 + } 240 + lb.Ref = bs.Ref 241 + lb.MimeType = bs.MimeType 242 + lb.Size = bs.Size 243 + if bs.Size < 0 { 244 + return xerrors.Errorf("parsing $blob CBOR: negative size: %d", bs.Size) 245 + } 246 + } else { 247 + legacy := LegacyBlob{} 248 + err := legacy.UnmarshalCBOR(bytes.NewReader(b)) 249 + if err != nil { 250 + return xerrors.Errorf("parsing legacy blob CBOR: %v", err) 251 + } 252 + refCid, err := cid.Decode(legacy.Cid) 253 + if err != nil { 254 + return xerrors.Errorf("parsing CID in legacy blob CBOR: %v", err) 255 + } 256 + lb.Ref = LexLink(refCid) 257 + lb.MimeType = legacy.MimeType 258 + lb.Size = -1 259 + } 260 + 261 + return nil 262 + }

-70

lex/util/util.go

··· 5 5 "encoding/json" 6 6 "fmt" 7 7 "io" 8 - 9 - "github.com/ipfs/go-cid" 10 8 ) 11 9 12 10 type typeExtractor struct { ··· 20 18 } 21 19 22 20 return te.Type, nil 23 - } 24 - 25 - type LegacyBlob struct { 26 - Cid string `json:"cid" cborgen:"cid"` 27 - MimeType string `json:"mimeType" cborgen:"mimeType"` 28 - } 29 - 30 - type CidLink struct { 31 - Cid string `json:"$link"` 32 - } 33 - 34 - type NewBlob struct { 35 - LexiconTypeID string `json:"$type,omitempty"` 36 - Ref CidLink `json:"ref" cborgen:"ref"` 37 - MimeType string `json:"mimeType" cborgen:"mimeType"` 38 - Size int64 `json:"size" cborgen:"size"` 39 - } 40 - 41 - type Blob struct { 42 - Ref cid.Cid `json:"ref" cborgen:"ref"` 43 - MimeType string `json:"mimeType" cborgen:"mimeType"` 44 - Size int64 `json:"size" cborgen:"size"` 45 - } 46 - 47 - func (b *Blob) MarshalJSON() ([]byte, error) { 48 - nb := NewBlob{ 49 - LexiconTypeID: "blob", 50 - Ref: CidLink{b.Ref.String()}, 51 - MimeType: b.MimeType, 52 - Size: b.Size, 53 - } 54 - return json.Marshal(nb) 55 - } 56 - 57 - func (b *Blob) UnmarshalJSON(raw []byte) error { 58 - typ, err := TypeExtract(raw) 59 - if err != nil { 60 - return fmt.Errorf("parsing blob: %v", err) 61 - } 62 - 63 - if typ == "blob" { 64 - var nb NewBlob 65 - err := json.Unmarshal(raw, &nb) 66 - if err != nil { 67 - return fmt.Errorf("parsing blob JSON: %v", err) 68 - } 69 - b.Ref, err = cid.Decode(nb.Ref.Cid) 70 - if err != nil { 71 - return fmt.Errorf("parsing blob CID: %v", err) 72 - } 73 - b.MimeType = nb.MimeType 74 - b.Size = nb.Size 75 - } else { 76 - var legacy *LegacyBlob 77 - err := json.Unmarshal(raw, legacy) 78 - if err != nil { 79 - return fmt.Errorf("parsing legacy blob: %v", err) 80 - } 81 - b.Ref, err = cid.Decode(legacy.Cid) 82 - if err != nil { 83 - return fmt.Errorf("parsing CID in legacy blob: %v", err) 84 - } 85 - b.MimeType = legacy.MimeType 86 - // TODO: copying the -1 here from atproto behavior. should verify if it 87 - // should be *size instead 88 - b.Size = -1 89 - } 90 - return nil 91 21 } 92 22 93 23 type CborChecker struct {

Configure Feed

Configure Feed