forked from
pds.ls/pdsls
atmosphere explorer
fix unpacking large repos on webkit
+105
-44
+3
-19
src/views/car/explore.tsx
+3
-19
src/views/car/explore.tsx
···
10
10
import { TextInput } from "../../components/text-input.jsx";
11
11
import { isTouchDevice } from "../../layout.jsx";
12
12
import { localDateFromTimestamp } from "../../utils/date.js";
13
+
import { createDropHandler, createFileChangeHandler, handleDragOver } from "./file-handlers.js";
13
14
import {
14
15
type Archive,
15
16
type CollectionEntry,
···
98
99
}
99
100
};
100
101
101
-
const handleFileChange = (e: Event) => {
102
-
const input = e.target as HTMLInputElement;
103
-
const file = input.files?.[0];
104
-
if (file) {
105
-
parseCarFile(file);
106
-
}
107
-
};
108
-
109
-
const handleDrop = (e: DragEvent) => {
110
-
e.preventDefault();
111
-
const file = e.dataTransfer?.files?.[0];
112
-
if (file && (file.name.endsWith(".car") || file.type === "application/vnd.ipld.car")) {
113
-
parseCarFile(file);
114
-
}
115
-
};
116
-
117
-
const handleDragOver = (e: DragEvent) => {
118
-
e.preventDefault();
119
-
};
102
+
const handleFileChange = createFileChangeHandler(parseCarFile);
103
+
const handleDrop = createDropHandler(parseCarFile);
120
104
121
105
const reset = () => {
122
106
setArchive(null);
+24
src/views/car/file-handlers.ts
+24
src/views/car/file-handlers.ts
···
1
+
export const isCarFile = (file: File): boolean => {
2
+
return file.name.endsWith(".car") || file.type === "application/vnd.ipld.car";
3
+
};
4
+
5
+
export const createFileChangeHandler = (onFile: (file: File) => void) => (e: Event) => {
6
+
const input = e.target as HTMLInputElement;
7
+
const file = input.files?.[0];
8
+
if (file) {
9
+
onFile(file);
10
+
}
11
+
input.value = "";
12
+
};
13
+
14
+
export const createDropHandler = (onFile: (file: File) => void) => (e: DragEvent) => {
15
+
e.preventDefault();
16
+
const file = e.dataTransfer?.files?.[0];
17
+
if (file && isCarFile(file)) {
18
+
onFile(file);
19
+
}
20
+
};
21
+
22
+
export const handleDragOver = (e: DragEvent) => {
23
+
e.preventDefault();
24
+
};
+78
-25
src/views/car/unpack.tsx
+78
-25
src/views/car/unpack.tsx
···
3
3
import { Title } from "@solidjs/meta";
4
4
import { FileSystemWritableFileStream, showSaveFilePicker } from "native-file-system-adapter";
5
5
import { createSignal, onCleanup } from "solid-js";
6
+
import { createDropHandler, createFileChangeHandler, handleDragOver } from "./file-handlers.js";
6
7
import { createLogger, LoggerView } from "./logger.jsx";
7
8
import { isIOS, toJsonValue, WelcomeView } from "./shared.jsx";
9
+
10
+
// Check if browser natively supports File System Access API
11
+
const hasNativeFileSystemAccess = "showSaveFilePicker" in window;
8
12
9
13
// HACK: Disable compression on WebKit due to an error being thrown
10
14
const isWebKit =
···
40
44
41
45
try {
42
46
let count = 0;
47
+
48
+
// On Safari/browsers without native File System Access API, use blob download
49
+
if (!hasNativeFileSystemAccess) {
50
+
const chunks: BlobPart[] = [];
51
+
52
+
const entryGenerator = async function* (): AsyncGenerator<ZipEntry> {
53
+
const progress = logger.progress(`Unpacking records (0 entries)`);
54
+
55
+
try {
56
+
for await (const entry of repo) {
57
+
if (signal.aborted) return;
58
+
59
+
try {
60
+
const record = toJsonValue(entry.record);
61
+
const filename = `${entry.collection}/${filenamify(entry.rkey)}.json`;
62
+
const data = JSON.stringify(record, null, 2);
63
+
64
+
yield { filename, data, compress: isWebKit ? false : "deflate" };
65
+
count++;
66
+
progress.update(`Unpacking records (${count} entries)`);
67
+
} catch {
68
+
// Skip entries with invalid data
69
+
}
70
+
}
71
+
} finally {
72
+
progress[Symbol.dispose]?.();
73
+
}
74
+
};
75
+
76
+
for await (const chunk of zip(entryGenerator())) {
77
+
if (signal.aborted) return;
78
+
chunks.push(chunk as BlobPart);
79
+
}
80
+
81
+
if (signal.aborted) return;
82
+
83
+
logger.log(`${count} records extracted`);
84
+
logger.log(`Creating download...`);
85
+
86
+
const blob = new Blob(chunks, { type: "application/zip" });
87
+
const url = URL.createObjectURL(blob);
88
+
const a = document.createElement("a");
89
+
a.href = url;
90
+
a.download = `${file.name.replace(/\.car$/, "")}.zip`;
91
+
document.body.appendChild(a);
92
+
a.click();
93
+
document.body.removeChild(a);
94
+
URL.revokeObjectURL(url);
95
+
96
+
logger.log(`Finished! Download started.`);
97
+
setPending(false);
98
+
return;
99
+
}
100
+
101
+
// Native File System Access API path
43
102
let writable: FileSystemWritableFileStream | undefined;
44
103
45
104
// Create async generator that yields ZipEntry as we read from CAR
···
70
129
if (err instanceof DOMException && err.name === "AbortError") {
71
130
logger.warn(`File picker was cancelled`);
72
131
} else {
73
-
console.warn(err);
74
132
logger.warn(`Something went wrong when opening the file picker`);
75
133
}
76
134
return undefined;
77
135
});
78
136
79
-
writable = await fd?.createWritable();
137
+
if (!fd) {
138
+
logger.warn(`No file handle obtained`);
139
+
return;
140
+
}
141
+
142
+
writable = await fd.createWritable();
80
143
81
144
if (writable === undefined) {
145
+
logger.warn(`Failed to create writable stream`);
82
146
return;
83
147
}
84
148
} finally {
···
116
180
return;
117
181
}
118
182
writeCount++;
119
-
if (writeCount % 100 !== 0) {
183
+
// Await every 100th write to apply backpressure
184
+
if (writeCount % 100 === 0) {
185
+
await writable.write(chunk);
186
+
} else {
120
187
writable.write(chunk); // Fire and forget
121
-
} else {
122
-
await writable.write(chunk); // Await periodically to apply backpressure
123
188
}
124
189
}
125
190
···
137
202
const flushProgress = logger.progress(`Flushing writes...`);
138
203
try {
139
204
await writable.close();
205
+
logger.log(`Finished! File saved successfully.`);
206
+
} catch (err) {
207
+
logger.error(`Failed to save file: ${err}`);
208
+
throw err; // Re-throw to be caught by outer catch
140
209
} finally {
141
210
flushProgress[Symbol.dispose]?.();
142
211
}
143
212
}
144
-
145
-
logger.log(`Finished!`);
146
213
} catch (err) {
147
214
if (signal.aborted) return;
148
-
console.error("Failed to unpack CAR file:", err);
149
215
logger.error(`Error: ${err}\nFile might be malformed, or might not be a CAR archive`);
150
216
} finally {
151
217
await repo?.dispose();
···
155
221
}
156
222
};
157
223
158
-
const handleFileChange = (e: Event) => {
159
-
const input = e.target as HTMLInputElement;
160
-
const file = input.files?.[0];
161
-
if (file) {
162
-
unpackToZip(file);
163
-
}
164
-
input.value = "";
165
-
};
224
+
const handleFileChange = createFileChangeHandler(unpackToZip);
166
225
226
+
// Wrap handleDrop to prevent multiple simultaneous uploads
227
+
const baseDrop = createDropHandler(unpackToZip);
167
228
const handleDrop = (e: DragEvent) => {
168
-
e.preventDefault();
169
229
if (pending()) return;
170
-
const file = e.dataTransfer?.files?.[0];
171
-
if (file && (file.name.endsWith(".car") || file.type === "application/vnd.ipld.car")) {
172
-
unpackToZip(file);
173
-
}
174
-
};
175
-
176
-
const handleDragOver = (e: DragEvent) => {
177
-
e.preventDefault();
230
+
baseDrop(e);
178
231
};
179
232
180
233
return (