mary.my.id / pkg-protobuf
protobuf codec with static type inference jsr.io/@mary/protobuf
typescript jsr
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

refactor!: properly handle packed/unpacked arrays

Mary 8c8c158f 6bfcc3bc

+237 -173
+74 -24
lib/mod.test.ts
··· 400 400 // #region Complex types 401 401 402 402 Deno.test('repeated fields', () => { 403 - const Message = p.message({ 404 - numbers: p.repeated(p.int32()), 405 - strings: p.repeated(p.string()), 406 - }, { 407 - numbers: 1, 408 - strings: 2, 409 - }); 410 - 411 403 const cases = [ 412 404 { 413 405 numbers: [], ··· 427 419 }, 428 420 ]; 429 421 430 - for (const data of cases) { 431 - const encoded = p.encode(Message, data); 432 - const decoded = p.decode(Message, encoded); 422 + { 423 + const Message = p.message({ 424 + numbers: p.repeated(p.int32(), false), 425 + strings: p.repeated(p.string(), false), 426 + }, { 427 + numbers: 1, 428 + strings: 2, 429 + }); 430 + 431 + for (const data of cases) { 432 + const encoded = p.encode(Message, data); 433 + const decoded = p.decode(Message, encoded); 434 + 435 + assertEquals(decoded, data); 436 + } 437 + } 438 + 439 + { 440 + const Message = p.message({ 441 + numbers: p.repeated(p.int32(), true), 442 + strings: p.repeated(p.string(), true), 443 + }, { 444 + numbers: 1, 445 + strings: 2, 446 + }); 433 447 434 - assertEquals(decoded, data); 448 + for (const data of cases) { 449 + const encoded = p.encode(Message, data); 450 + const decoded = p.decode(Message, encoded); 451 + 452 + assertEquals(decoded, data); 453 + } 435 454 } 436 455 }); 437 456 ··· 610 629 name: 2, 611 630 }); 612 631 632 + const PersonMap = p.map(p.string(), Person); 633 + type PersonMap = p.InferInput<typeof PersonMap>; 634 + 613 635 const Message = p.message({ 614 - map: p.map(p.string(), Person), 636 + map: PersonMap, 615 637 }, { 616 638 map: 1, 617 639 }); 618 640 619 - const cases = [ 620 - new Map(), 621 - new Map([['item1', { id: 1, name: 'first' }]]), 622 - new Map([['item1', { id: 1, name: 'first' }], ['item2', { id: 2, name: 'second' }]]), 623 - ]; 641 + { 642 + const map: PersonMap = []; 643 + 644 + const encoded = p.encode(Message, { map }); 645 + const decoded = p.decode(Message, encoded); 624 646 625 - for (const map of cases) { 647 + assertEquals(decoded, { map }); 648 + } 649 + 650 + { 651 + const map: PersonMap = [{ key: 'item1', value: { id: 1, name: 'first' } }]; 652 + 653 + const encoded = p.encode(Message, { map }); 654 + const decoded = p.decode(Message, encoded); 655 + 656 + assertEquals(decoded, { map }); 657 + } 658 + 659 + { 660 + const map: PersonMap = [ 661 + { key: 'item1', value: { id: 1, name: 'first' } }, 662 + { key: 'item2', value: { id: 2, name: 'second' } }, 663 + ]; 664 + 626 665 const encoded = p.encode(Message, { map }); 627 666 const decoded = p.decode(Message, encoded); 628 667 ··· 1082 1121 }); 1083 1122 1084 1123 Deno.test('very large arrays', () => { 1085 - const Message = p.message({ strings: p.repeated(p.string()) }, { strings: 1 }); 1124 + const strings = Array.from({ length: 10000 }, () => nanoid(8)); 1125 + 1126 + { 1127 + const Message = p.message({ strings: p.repeated(p.string(), false) }, { strings: 1 }); 1128 + 1129 + const encoded = p.encode(Message, { strings }); 1130 + const decoded = p.decode(Message, encoded); 1131 + 1132 + assertEquals(decoded, { strings }); 1133 + } 1086 1134 1087 - const strings = Array.from({ length: 10000 }, () => nanoid(8)); 1135 + { 1136 + const Message = p.message({ strings: p.repeated(p.string(), true) }, { strings: 1 }); 1088 1137 1089 - const encoded = p.encode(Message, { strings }); 1090 - const decoded = p.decode(Message, encoded); 1138 + const encoded = p.encode(Message, { strings }); 1139 + const decoded = p.decode(Message, encoded); 1091 1140 1092 - assertEquals(decoded, { strings }); 1141 + assertEquals(decoded, { strings }); 1142 + } 1093 1143 }); 1094 1144 1095 1145 Deno.test('large string handling', () => {
+159 -149
lib/mod.ts
··· 17 17 | 'bigint' 18 18 | 'boolean' 19 19 | 'bytes' 20 - | 'map' 21 20 | 'number' 22 21 | 'object' 23 22 | 'string'; ··· 55 54 const BIGINT_TYPE_ISSUE: IssueLeaf = { ok: false, code: 'invalid_type', expected: 'bigint' }; 56 55 const BOOLEAN_TYPE_ISSUE: IssueLeaf = { ok: false, code: 'invalid_type', expected: 'boolean' }; 57 56 const BYTES_TYPE_ISSUE: IssueLeaf = { ok: false, code: 'invalid_type', expected: 'bytes' }; 58 - const MAP_TYPE_ISSUE: IssueLeaf = { ok: false, code: 'invalid_type', expected: 'map' }; 59 57 const NUMBER_TYPE_ISSUE: IssueLeaf = { ok: false, code: 'invalid_type', expected: 'number' }; 60 58 const OBJECT_TYPE_ISSUE: IssueLeaf = { ok: false, code: 'invalid_type', expected: 'object' }; 61 59 const STRING_TYPE_ISSUE: IssueLeaf = { ok: false, code: 'invalid_type', expected: 'string' }; ··· 501 499 502 500 type RawResult<T = unknown> = Ok<T> | IssueTree; 503 501 504 - type Decoder = (this: void, state: DecoderState) => RawResult; 502 + type Decoder = (state: DecoderState) => RawResult; 505 503 506 - type Encoder = (this: void, state: EncoderState, input: unknown) => IssueTree | void; 504 + type Encoder = (state: EncoderState, input: unknown) => IssueTree | void; 507 505 508 506 export interface BaseSchema<TInput = unknown, TOutput = TInput> { 509 507 readonly kind: 'schema'; ··· 1236 1234 }; 1237 1235 1238 1236 // #region Repeated schema 1239 - export interface RepeatedSchema<TItem extends BaseSchema> extends BaseSchema<unknown[]> { 1237 + export interface RepeatedSchema<TItem extends BaseSchema = BaseSchema> extends BaseSchema<unknown[]> { 1240 1238 readonly type: 'repeated'; 1241 - readonly wire: 2; 1239 + readonly packed: boolean; 1240 + readonly wire: WireType; 1242 1241 readonly item: TItem; 1243 1242 1244 1243 readonly [kObjectType]?: { in: InferInput<TItem>[]; out: InferOutput<TItem>[] }; ··· 1250 1249 * @returns repeated schema 1251 1250 */ 1252 1251 // #__NO_SIDE_EFFECTS__ 1253 - export const repeated = <TItem extends BaseSchema>(item: TItem | (() => TItem)): RepeatedSchema<TItem> => { 1252 + export const repeated = <TItem extends BaseSchema>( 1253 + item: TItem | (() => TItem), 1254 + packed = false, // Default to non-packed for compatibility 1255 + ): RepeatedSchema<TItem> => { 1254 1256 const resolvedShape = lazy(() => { 1255 1257 return typeof item === 'function' ? item() : item; 1256 1258 }); ··· 1258 1260 return { 1259 1261 kind: 'schema', 1260 1262 type: 'repeated', 1261 - wire: 2, 1263 + packed: packed, 1264 + get wire() { 1265 + return lazyProperty(this, 'wire', resolvedShape.value.wire); 1266 + }, 1262 1267 get item() { 1263 1268 return lazyProperty(this, 'item', resolvedShape.value); 1264 1269 }, ··· 1266 1271 const shape = resolvedShape.value; 1267 1272 1268 1273 const decoder: Decoder = (state) => { 1269 - const length = readVarint(state); 1270 - if (!length.ok) { 1271 - return length; 1272 - } 1273 - 1274 - const bytes = readBytes(state, length.value); 1275 - if (!bytes.ok) { 1276 - return bytes; 1277 - } 1278 - 1279 - const children: DecoderState = { 1280 - b: bytes.value, 1281 - p: 0, 1282 - v: null, 1283 - }; 1284 - 1285 - const array: any[] = []; 1286 - 1287 - let idx = 0; 1288 - let issues: IssueTree | undefined; 1289 - 1290 - while (children.p < length.value) { 1291 - const r = shape['~decode'](children); 1292 - 1293 - if (!r.ok) { 1294 - return prependPath(idx, r); 1295 - } 1296 - 1297 - array.push(r.value); 1298 - idx++; 1299 - } 1300 - 1301 - if (issues !== undefined) { 1302 - return issues; 1303 - } 1304 - 1305 - return { ok: true, value: array }; 1274 + return lazyProperty(this, '~decode', shape['~decode'])(state); 1306 1275 }; 1307 1276 1308 1277 return lazyProperty(this, '~decode', decoder); ··· 1311 1280 const shape = resolvedShape.value; 1312 1281 1313 1282 const encoder: Encoder = (state, input) => { 1314 - if (!Array.isArray(input)) { 1315 - return ARRAY_TYPE_ISSUE; 1316 - } 1317 - 1318 - const children: EncoderState = { 1319 - c: [], 1320 - b: new Uint8Array(CHUNK_SIZE), 1321 - v: null, 1322 - p: 0, 1323 - l: 0, 1324 - }; 1325 - 1326 - for (let idx = 0, len = input.length; idx < len; idx++) { 1327 - const result = shape['~encode'](children, input[idx]); 1328 - 1329 - if (result) { 1330 - return prependPath(idx, result); 1331 - } 1332 - } 1333 - 1334 - const packed = finishEncode(children); 1335 - 1336 - writeVarint(state, packed.length); 1337 - writeBytes(state, packed); 1283 + return lazyProperty(this, '~encode', shape['~encode'])(state, input); 1338 1284 }; 1339 1285 1340 1286 return lazyProperty(this, '~encode', encoder); 1341 1287 }, 1342 1288 }; 1289 + }; 1290 + 1291 + const isRepeatedSchema = (schema: BaseSchema): schema is RepeatedSchema<any> => { 1292 + return schema.type === 'repeated'; 1343 1293 }; 1344 1294 1345 1295 // #region Optional schema ··· 1382 1332 type: 'optional', 1383 1333 wrapped: wrapped, 1384 1334 default: defaultValue, 1385 - wire: wrapped.wire, 1335 + get 'wire'() { 1336 + return lazyProperty(this, 'wire', wrapped.wire); 1337 + }, 1386 1338 '~decode'(state) { 1387 - return wrapped['~decode'](state); 1339 + return lazyProperty(this, '~decode', wrapped['~decode'])(state); 1388 1340 }, 1389 1341 '~encode'(state, input) { 1390 - return wrapped['~encode'](state, input); 1342 + return lazyProperty(this, '~encode', wrapped['~encode'])(state, input); 1391 1343 }, 1392 1344 }; 1393 1345 }; ··· 1457 1409 wire: WireType; 1458 1410 1459 1411 optional: boolean; 1412 + repeated: boolean; 1413 + packed: boolean; 1460 1414 1461 1415 wireIssue: IssueTree; 1462 1416 missingIssue: IssueTree; ··· 1494 1448 const schema = obj[key]; 1495 1449 const tag = tags[key]; 1496 1450 1451 + let innerSchema = schema; 1452 + 1453 + const isOptional = isOptionalSchema(innerSchema); 1454 + if (isOptional) { 1455 + innerSchema = (innerSchema as OptionalSchema).wrapped; 1456 + } 1457 + 1458 + const isRepeated = isRepeatedSchema(innerSchema); 1459 + const isPacked = isRepeated && (innerSchema as RepeatedSchema).packed; 1460 + if (isRepeated) { 1461 + innerSchema = (innerSchema as RepeatedSchema).item; 1462 + } 1463 + 1497 1464 resolved[tag] = { 1498 1465 key: key, 1499 1466 schema: schema, 1500 1467 tag: tag, 1501 1468 wire: schema.wire, 1502 - optional: isOptionalSchema(schema), 1469 + optional: isOptional, 1470 + repeated: isRepeated, 1471 + packed: isPacked, 1503 1472 wireIssue: prependPath(key, { ok: false, code: 'invalid_wire', expected: schema.wire }), 1504 1473 missingIssue: prependPath(key, ISSUE_MISSING), 1505 1474 }; ··· 1536 1505 let seenCount = 0; 1537 1506 1538 1507 const obj: Record<string, unknown> = {}; 1539 - let issues: IssueTree | undefined; 1540 1508 1541 1509 const end = state.b.length; 1542 1510 while (state.p < end) { ··· 1572 1540 return entry.wireIssue; 1573 1541 } 1574 1542 1575 - // Decode the value 1576 - const result = entry.schema['~decode'](state); 1543 + const schema = entry.schema; 1544 + const key = entry.key; 1577 1545 1578 - // Failed to decode, file an issue 1579 - if (!result.ok) { 1580 - return prependPath(entry.key, result); 1581 - } 1546 + if (entry.repeated) { 1547 + if (entry.packed) { 1548 + const array: unknown[] = []; 1549 + 1550 + const length = readVarint(state); 1551 + if (!length.ok) { 1552 + return prependPath(key, length); 1553 + } 1582 1554 1583 - /*#__INLINE__*/ set(obj, entry.key, result.value); 1555 + const bytes = readBytes(state, length.value); 1556 + if (!bytes.ok) { 1557 + return prependPath(key, bytes); 1558 + } 1559 + 1560 + const children: DecoderState = { 1561 + b: bytes.value, 1562 + p: 0, 1563 + v: null, 1564 + }; 1565 + 1566 + let idx = 0; 1567 + while (children.p < length.value) { 1568 + const r = schema['~decode'](children); 1569 + 1570 + if (!r.ok) { 1571 + return prependPath(key, prependPath(idx, r)); 1572 + } 1573 + 1574 + array.push(r.value); 1575 + idx++; 1576 + } 1577 + 1578 + /*#__INLINE__*/ set(obj, key, array); 1579 + } else { 1580 + let array = obj[key] as unknown[] | undefined; 1581 + if (array === undefined) { 1582 + set(obj, key, array = []); 1583 + } 1584 + 1585 + const result = schema['~decode'](state); 1586 + 1587 + if (!result.ok) { 1588 + return prependPath(key, prependPath(array.length, result)); 1589 + } 1590 + 1591 + array.push(result.value); 1592 + } 1593 + } else { 1594 + const result = schema['~decode'](state); 1595 + 1596 + if (!result.ok) { 1597 + return prependPath(key, result); 1598 + } 1599 + 1600 + /*#__INLINE__*/ set(obj, key, result.value); 1601 + } 1584 1602 } 1585 1603 1586 1604 if (seenCount < len) { ··· 1599 1617 1600 1618 /*#__INLINE__*/ set(obj, entry.key, defaultValue); 1601 1619 } 1620 + } else if (entry.repeated && !entry.packed) { 1621 + /*#__INLINE__*/ set(obj, entry.key, []); 1602 1622 } else { 1603 1623 return entry.missingIssue; 1604 1624 } 1605 1625 } 1606 1626 } 1607 - } 1608 - 1609 - if (issues !== undefined) { 1610 - return issues; 1611 1627 } 1612 1628 1613 1629 return { ok: true, value: obj }; ··· 1654 1670 const entry = shape[tag]; 1655 1671 const fieldValue = obj[entry.key]; 1656 1672 1657 - if (entry.optional && fieldValue === undefined) { 1673 + if (fieldValue === undefined && entry.optional) { 1658 1674 continue; 1659 1675 } 1660 1676 1661 - writeVarint(state, (entry.tag << 3) | entry.wire); 1677 + const schema = entry.schema; 1678 + const key = entry.key; 1679 + 1680 + if (entry.repeated) { 1681 + if (!Array.isArray(fieldValue)) { 1682 + return prependPath(key, ARRAY_TYPE_ISSUE); 1683 + } 1684 + 1685 + if (entry.packed) { 1686 + const children: EncoderState = { 1687 + c: [], 1688 + b: new Uint8Array(CHUNK_SIZE), 1689 + v: null, 1690 + p: 0, 1691 + l: 0, 1692 + }; 1662 1693 1663 - const result = entry.schema['~encode'](state, fieldValue); 1694 + for (let idx = 0, len = fieldValue.length; idx < len; idx++) { 1695 + const result = schema['~encode'](children, fieldValue[idx]); 1664 1696 1665 - if (result) { 1666 - return prependPath(entry.key, result); 1697 + if (result) { 1698 + return prependPath(idx, result); 1699 + } 1700 + } 1701 + 1702 + const buffer = finishEncode(children); 1703 + 1704 + writeVarint(state, (entry.tag << 3) | entry.wire); 1705 + writeVarint(state, buffer.length); 1706 + writeBytes(state, buffer); 1707 + } else { 1708 + for (let idx = 0, len = fieldValue.length; idx < len; idx++) { 1709 + writeVarint(state, (entry.tag << 3) | entry.wire); 1710 + const result = schema['~encode'](state, fieldValue[idx]); 1711 + 1712 + if (result) { 1713 + return prependPath(idx, result); 1714 + } 1715 + } 1716 + } 1717 + } else { 1718 + writeVarint(state, (entry.tag << 3) | entry.wire); 1719 + const result = schema['~encode'](state, fieldValue); 1720 + 1721 + if (result) { 1722 + return prependPath(key, result); 1723 + } 1667 1724 } 1668 1725 } 1669 1726 }; ··· 1715 1772 1716 1773 export type MapValueSchema = BaseSchema; 1717 1774 1718 - export interface MapSchema<TKey extends MapKeySchema, TValue extends MapValueSchema> 1719 - extends BaseSchema<unknown[]> { 1720 - readonly type: 'map'; 1721 - readonly wire: 2; 1722 - readonly key: TKey; 1723 - readonly value: TValue; 1724 - 1725 - readonly [kObjectType]?: { 1726 - in: Map<InferInput<TKey>, InferInput<TValue>>; 1727 - out: Map<InferOutput<TKey>, InferOutput<TValue>>; 1728 - }; 1729 - } 1775 + export interface MapSchema<TKey extends MapKeySchema, TValue extends MapValueSchema> extends 1776 + RepeatedSchema< 1777 + MessageSchema<{ 1778 + key: TKey; 1779 + value: TValue; 1780 + }, { 1781 + readonly key: 1; 1782 + readonly value: 2; 1783 + }> 1784 + > {} 1730 1785 1731 1786 /** 1732 1787 * creates a key-value map schema ··· 1737 1792 export const map = <TKey extends MapKeySchema, TValue extends MapValueSchema>( 1738 1793 key: TKey, 1739 1794 value: TValue, 1795 + packed = false, 1740 1796 ): MapSchema<TKey, TValue> => { 1741 - const Schema = repeated(message({ key, value }, { key: 1, value: 2 })); 1742 - 1743 - type Entry = { key: TKey; value: TValue }; 1744 - 1745 - return { 1746 - kind: 'schema', 1747 - type: 'map', 1748 - wire: 2, 1749 - key, 1750 - value, 1751 - get '~decode'() { 1752 - const decoder: Decoder = (state) => { 1753 - const result = Schema['~decode'](state); 1754 - if (!result.ok) { 1755 - return result; 1756 - } 1757 - 1758 - const map = new Map(); 1759 - 1760 - const entries = result.value as Entry[]; 1761 - for (let idx = 0, len = entries.length; idx < len; idx++) { 1762 - const entry = entries[idx]; 1763 - map.set(entry.key, entry.value); 1764 - } 1765 - 1766 - return { ok: true, value: map }; 1767 - }; 1768 - 1769 - return lazyProperty(this, '~decode', decoder); 1770 - }, 1771 - get '~encode'() { 1772 - const encoder: Encoder = (state, input) => { 1773 - if (!(input instanceof Map)) { 1774 - return MAP_TYPE_ISSUE; 1775 - } 1776 - 1777 - const entries: Entry[] = []; 1778 - for (const [key, value] of input) { 1779 - entries.push({ key, value }); 1780 - } 1781 - 1782 - return Schema['~encode'](state, entries); 1783 - }; 1784 - 1785 - return lazyProperty(this, '~encode', encoder); 1786 - }, 1787 - }; 1797 + return repeated(message({ key, value }, { key: 1, value: 2 }), packed); 1788 1798 }; 1789 1799 1790 1800 // #endregion
+4
lib/utils.ts
··· 80 80 81 81 return result.written || 0; 82 82 }; 83 + 84 + export const assertUnreachable = (): never => { 85 + throw new Error(`not implemented`); 86 + };