a collection of lightweight TypeScript packages for AT Protocol, the protocol powering Bluesky
atproto
bluesky
typescript
npm
refactor(varint)!: faster varint decode
+97
-32
+8
.changeset/eager-windows-sit.md
+8
.changeset/eager-windows-sit.md
+7
.changeset/fair-ads-shave.md
+7
.changeset/fair-ads-shave.md
+8
.changeset/good-gifts-smile.md
+8
.changeset/good-gifts-smile.md
+7
.changeset/new-singers-design.md
+7
.changeset/new-singers-design.md
+8
-5
packages/utilities/car/lib/reader.ts
+8
-5
packages/utilities/car/lib/reader.ts
···
7
7
8
8
interface SyncByteReader {
9
9
readonly pos: number;
10
+
readonly source: Uint8Array;
10
11
upto(size: number): Uint8Array;
11
12
exactly(size: number, seek: boolean): Uint8Array;
12
13
seek(size: number): void;
···
72
73
get pos() {
73
74
return pos;
74
75
},
76
+
get source() {
77
+
return buf;
78
+
},
75
79
76
80
seek(size) {
77
81
if (size > buf.length - pos) {
···
99
103
};
100
104
101
105
const readVarint = (reader: SyncByteReader, size: number): number => {
102
-
const buf = reader.upto(size);
103
-
if (buf.length === 0) {
106
+
if (reader.pos >= reader.source.length) {
104
107
throw new RangeError(`unexpected end of data`);
105
108
}
106
109
107
-
const [int, read] = varint.decode(buf);
108
-
reader.seek(read);
110
+
const { value, nextOffset } = varint.decode(reader.source, reader.pos, size);
111
+
reader.seek(nextOffset - reader.pos);
109
112
110
-
return int;
113
+
return value;
111
114
};
112
115
113
116
const readHeader = (reader: SyncByteReader): CarHeader => {
+3
-3
packages/utilities/varint/README.md
+3
-3
packages/utilities/varint/README.md
···
16
16
// -> bytesWritten: 2
17
17
18
18
// decoding
19
-
const [num, bytesRead] = decode(encoded);
20
-
// -> num: 420
21
-
// -> bytesRead: 2
19
+
const { value, nextOffset } = decode(encoded);
20
+
// -> value: 420
21
+
// -> nextOffset: 2
22
22
23
23
// check encoding length beforehand
24
24
encodingLength(420); // -> 2
+23
-3
packages/utilities/varint/lib/index.test.ts
+23
-3
packages/utilities/varint/lib/index.test.ts
···
13
13
const encoded: number[] = [];
14
14
const encodedLength = encode(expected, encoded);
15
15
16
-
const [actual, actualLength] = decode(encoded);
16
+
const { value, nextOffset } = decode(encoded);
17
17
18
-
expect(actual).toBe(expected);
19
-
expect(actualLength).toBe(encodedLength);
18
+
expect(value).toBe(expected);
19
+
expect(nextOffset).toBe(encodedLength);
20
20
}
21
21
});
22
22
23
23
describe('encode', () => {
24
24
it('throws on very large numbers', () => {
25
25
expect(() => encode(2 ** 54 - 1, [])).toThrow();
26
+
});
27
+
});
28
+
29
+
describe('decode', () => {
30
+
it('supports decoding at offset', () => {
31
+
const encoded: number[] = [255, 255, 255];
32
+
const written = encode(420, encoded, 1);
33
+
34
+
const { value, nextOffset } = decode(encoded, 1);
35
+
36
+
expect(value).toBe(420);
37
+
expect(nextOffset).toBe(1 + written);
38
+
});
39
+
40
+
it('respects length', () => {
41
+
const encoded: number[] = [];
42
+
encode(16384, encoded);
43
+
44
+
expect(() => decode(encoded, 0, 2)).toThrow();
45
+
expect(decode(encoded, 0, 3)).toEqual({ value: 16384, nextOffset: 3 });
26
46
});
27
47
});
28
48
+33
-21
packages/utilities/varint/lib/index.ts
+33
-21
packages/utilities/varint/lib/index.ts
···
13
13
const N8 = 2 ** 56;
14
14
const N9 = 2 ** 63;
15
15
16
+
const MAX_SAFE_INTEGER = Number.MAX_SAFE_INTEGER;
17
+
18
+
const _min = Math.min;
19
+
20
+
export interface DecodeResult {
21
+
/** decoded number */
22
+
value: number;
23
+
/** position immediately after the varint */
24
+
nextOffset: number;
25
+
}
26
+
16
27
/**
17
-
* Encodes a varint
28
+
* encodes a varint
18
29
* @param num Number to encode
19
30
* @param buf Buffer to write on
20
31
* @param offset Starting position on the buffer
21
32
* @returns The amount of bytes written
22
33
*/
23
-
export const encode = (num: number, buf: Uint8Array | number[], offset = 0): number => {
24
-
if (num > Number.MAX_SAFE_INTEGER) {
34
+
export const encode = (num: number, buf: Uint8Array, offset = 0): number => {
35
+
if (num > MAX_SAFE_INTEGER) {
25
36
throw new RangeError('could not encode varint');
26
37
}
27
38
···
77
88
};
78
89
79
90
/**
80
-
* Decodes a varint
81
-
* @param buf Buffer to read from
82
-
* @param offset Starting position on the buffer
83
-
* @returns A tuple containing the resulting number, and the amount of bytes read
91
+
* decodes a varint and returns the value with the next byte offset
92
+
* @param buf buffer to read from
93
+
* @param offset starting position on the buffer
94
+
* @param length maximum bytes to consume from offset
95
+
* @returns decoded value and the next offset
84
96
*/
85
-
export const decode = (buf: Uint8Array | number[], offset = 0): [num: number, read: number] => {
86
-
const l = buf.length;
97
+
export const decode = (buf: Uint8Array, offset = 0, length = buf.length): DecodeResult => {
98
+
const end = _min(offset + length, buf.length);
87
99
let counter = offset;
88
100
89
-
if (counter >= l) {
101
+
if (counter >= end) {
90
102
throw new RangeError('could not decode varint');
91
103
}
92
104
93
105
let b = buf[counter++];
94
106
let res = b & REST;
95
107
if (b < MSB) {
96
-
return [res, 1];
108
+
return { value: res, nextOffset: counter };
97
109
}
98
110
99
-
if (counter >= l) {
111
+
if (counter >= end) {
100
112
throw new RangeError('could not decode varint');
101
113
}
102
114
103
115
b = buf[counter++];
104
116
res |= (b & REST) << 7;
105
117
if (b < MSB) {
106
-
return [res, 2];
118
+
return { value: res, nextOffset: counter };
107
119
}
108
120
109
-
if (counter >= l) {
121
+
if (counter >= end) {
110
122
throw new RangeError('could not decode varint');
111
123
}
112
124
113
125
b = buf[counter++];
114
126
res |= (b & REST) << 14;
115
127
if (b < MSB) {
116
-
return [res, 3];
128
+
return { value: res, nextOffset: counter };
117
129
}
118
130
119
-
if (counter >= l) {
131
+
if (counter >= end) {
120
132
throw new RangeError('could not decode varint');
121
133
}
122
134
123
135
b = buf[counter++];
124
136
res |= (b & REST) << 21;
125
137
if (b < MSB) {
126
-
return [res, 4];
138
+
return { value: res, nextOffset: counter };
127
139
}
128
140
129
-
if (counter >= l) {
141
+
if (counter >= end) {
130
142
throw new RangeError('could not decode varint');
131
143
}
132
144
133
145
b = buf[counter++];
134
146
res += (b & REST) * N4;
135
147
if (b < MSB) {
136
-
return [res, 5];
148
+
return { value: res, nextOffset: counter };
137
149
}
138
150
139
151
let shift = 35;
140
152
do {
141
-
if (counter >= l) {
153
+
if (counter >= end) {
142
154
throw new RangeError('could not decode varint');
143
155
}
144
156
···
147
159
shift += 7;
148
160
} while (b >= MSB);
149
161
150
-
return [res, counter - offset];
162
+
return { value: res, nextOffset: counter };
151
163
};
152
164
153
165
/**