handy online tools for AT Protocol
boat.kelinci.net
atproto
bluesky
atcute
typescript
solidjs
refactor: use streamed repo reader
+18
-18
+3
src/main.tsx
+3
src/main.tsx
···
22
22
if (Symbol.dispose === undefined) {
23
23
Object.defineProperty(Symbol, 'dispose', { value: Symbol.for(`Symbol.dispose`) });
24
24
}
25
+
if (Symbol.asyncDispose === undefined) {
26
+
Object.defineProperty(Symbol, 'asyncDispose', { value: Symbol.for(`Symbol.asyncDispose`) });
27
+
}
25
28
26
29
render(App, document.body);
+7
-3
src/views/repository/repo-archive-explore/page.tsx
+7
-3
src/views/repository/repo-archive-explore/page.tsx
···
1
1
import { Match, Switch } from 'solid-js';
2
2
3
-
import { iterateAtpRepo } from '@atcute/car';
3
+
import { RepoReader } from '@atcute/car/v4';
4
4
5
5
import { createMutation } from '~/lib/utils/mutation';
6
6
···
12
12
const ArchiveExplorePage = () => {
13
13
const mutation = createMutation({
14
14
async mutationFn({ file }: { file: File }): Promise<Archive> {
15
-
const buffer = new Uint8Array(await file.arrayBuffer());
15
+
const stream = file.stream();
16
+
await using repo = RepoReader.fromStream(stream);
16
17
17
18
const collections = new Map<string, RecordEntry[]>();
18
19
const archive: Archive = {
···
20
21
entries: [],
21
22
};
22
23
23
-
for (const entry of iterateAtpRepo(buffer)) {
24
+
for await (const entry of repo) {
24
25
let list = collections.get(entry.collection);
25
26
if (list === undefined) {
26
27
collections.set(entry.collection, (list = []));
···
41
42
}
42
43
43
44
return archive;
45
+
},
46
+
onError(err) {
47
+
console.error(err);
44
48
},
45
49
});
46
50
+8
-15
src/views/repository/repo-archive-unpack.tsx
+8
-15
src/views/repository/repo-archive-unpack.tsx
···
1
1
import { FileSystemWritableFileStream, showSaveFilePicker } from 'native-file-system-adapter';
2
2
import { createSignal } from 'solid-js';
3
3
4
-
import { iterateAtpRepo } from '@atcute/car';
4
+
import { RepoReader } from '@atcute/car/v4';
5
5
import { writeTarEntry } from '@mary/tar';
6
6
7
7
import { createDropZone } from '~/lib/hooks/dropzone';
···
39
39
});
40
40
41
41
const mutate = async (file: File, signal: AbortSignal) => {
42
-
logger.info(`Starting extraction for ${file.name}`);
42
+
logger.log(`Starting extraction`);
43
43
44
-
const buf = await file.arrayBuffer();
45
-
const ui8 = new Uint8Array(buf);
44
+
const stream = file.stream();
45
+
await using repo = RepoReader.fromStream(stream);
46
46
47
-
let currentCollection: string | undefined;
48
47
let count = 0;
49
48
50
49
let writable: FileSystemWritableFileStream | undefined;
51
50
52
-
for (const { collection, rkey, record } of iterateAtpRepo(ui8)) {
51
+
using progress = logger.progress(`Unpacking records (${count} entries)`);
52
+
53
+
for await (const { collection, rkey, record } of repo) {
53
54
if (writable === undefined) {
54
55
using _progress = logger.progress(`Waiting for the user`);
55
56
···
87
88
88
89
signal.throwIfAborted();
89
90
90
-
if (currentCollection !== collection) {
91
-
logger.log(`Current progress: ${collection}`);
92
-
currentCollection = collection;
93
-
94
-
if (yieldToScheduler === undefined) {
95
-
await yieldToIdle();
96
-
}
97
-
}
98
-
99
91
const entry = writeTarEntry({
100
92
filename: `${collection}/${filenamify(rkey)}.json`,
101
93
data: JSON.stringify(record, null, 2),
102
94
});
103
95
96
+
progress.update(`Unpacking records (${count} entries)`);
104
97
writable.write(entry);
105
98
count++;
106
99