Small
wget like mirroring utility.
Fix path handling for pages with child resources and add JSON output logging
+134
-61
+134
-61
src/cli.ts
+134
-61
src/cli.ts
···
1
1
#!/usr/bin/env node
2
2
import { cli, define } from 'gunshi';
3
3
import { HTMLRewriter } from 'html-rewriter-wasm';
4
-
import { access, mkdir, readFile, writeFile } from 'node:fs/promises';
5
-
import { dirname, join } from 'node:path';
4
+
import { access, mkdir, stat, writeFile } from 'node:fs/promises';
5
+
import { dirname, join, resolve } from 'node:path';
6
6
7
7
type OverwriteMode = 'ignore' | 'cache' | 'update';
8
8
9
9
type LinkPredicate = (url: URL, baseUrl: URL) => boolean;
10
+
11
+
interface PageStats {
12
+
url: string;
13
+
path: string;
14
+
size: number;
15
+
links: number;
16
+
assets: number;
17
+
cached: boolean;
18
+
}
10
19
11
20
function originPredicate(baseUrl: URL): LinkPredicate {
12
21
return (url) => url.origin === baseUrl.origin;
···
23
32
return parts.length > 0 ? parts[parts.length - 1] : url.hostname;
24
33
}
25
34
26
-
async function scrapePage(url: URL, outputDir: string, predicate: LinkPredicate, queue: Set<string>, processed: Set<string>, overwrite: OverwriteMode) {
27
-
if (processed.has(url.href)) return;
28
-
29
-
processed.add(url.href);
30
-
31
-
let filePath = url.pathname;
32
-
if (filePath.endsWith('/') || filePath === '') {
33
-
filePath = filePath + 'index.html';
34
-
} else if (!filePath.endsWith('.html') && !filePath.endsWith('.htm')) {
35
-
filePath = filePath + '.html';
36
-
}
37
-
38
-
const outputPath = join(outputDir, url.hostname, filePath);
39
-
40
-
let html: string;
35
+
async function logFileStats(stats: PageStats) {
36
+
console.log(JSON.stringify(stats));
37
+
}
38
+
39
+
async function downloadAsset(url: URL, outputDir: string, overwrite: OverwriteMode): Promise<boolean> {
40
+
const assetOutputPath = join(outputDir, url.hostname, url.pathname);
41
41
42
42
try {
43
-
await access(outputPath);
43
+
await access(assetOutputPath);
44
44
45
45
if (overwrite === 'ignore') {
46
-
return;
46
+
const assetStats = await stat(assetOutputPath);
47
+
logFileStats({
48
+
url: url.href,
49
+
path: assetOutputPath,
50
+
size: assetStats.size,
51
+
links: 0,
52
+
assets: 0,
53
+
cached: true,
54
+
});
55
+
return true;
47
56
}
48
57
49
58
if (overwrite === 'cache') {
50
-
html = await readFile(outputPath, 'utf-8');
51
-
} else {
52
-
const response = await fetch(url.href);
53
-
if (!response.ok) {
54
-
return;
55
-
}
56
-
html = await response.text();
57
-
await writeFile(outputPath, html);
59
+
const assetStats = await stat(assetOutputPath);
60
+
logFileStats({
61
+
url: url.href,
62
+
path: assetOutputPath,
63
+
size: assetStats.size,
64
+
links: 0,
65
+
assets: 0,
66
+
cached: true,
67
+
});
68
+
return true;
58
69
}
59
-
} catch {
60
-
const response = await fetch(url.href);
61
-
if (!response.ok) {
62
-
return;
63
-
}
64
-
html = await response.text();
65
-
await mkdir(dirname(outputPath), { recursive: true });
66
-
await writeFile(outputPath, html);
70
+
} catch {}
71
+
72
+
const response = await fetch(url.href);
73
+
if (!response.ok) {
74
+
return false;
75
+
}
76
+
77
+
const assetData = await response.arrayBuffer();
78
+
await mkdir(dirname(assetOutputPath), { recursive: true });
79
+
await writeFile(assetOutputPath, Buffer.from(assetData));
80
+
81
+
logFileStats({
82
+
url: url.href,
83
+
path: assetOutputPath,
84
+
size: assetData.byteLength,
85
+
links: 0,
86
+
assets: 0,
87
+
cached: false,
88
+
});
89
+
90
+
return true;
91
+
}
92
+
93
+
async function scrapePage(url: URL, outputDir: string, predicate: LinkPredicate, queue: Set<string>, processed: Set<string>, overwrite: OverwriteMode): Promise<void> {
94
+
const urlWithoutHash = url.href.split('#')[0];
95
+
if (processed.has(urlWithoutHash)) return;
96
+
97
+
processed.add(urlWithoutHash);
98
+
99
+
const response = await fetch(url.href);
100
+
if (!response.ok) {
101
+
return;
67
102
}
103
+
104
+
const html = await response.text();
68
105
69
106
const links = new Set<string>();
70
107
const assets = new Set<string>();
···
78
115
79
116
try {
80
117
const linkUrl = new URL(href, url);
81
-
if (predicate(linkUrl, url) && !processed.has(linkUrl.href)) {
118
+
if (predicate(linkUrl, url) && !processed.has(linkUrl.href.split('#')[0])) {
82
119
links.add(linkUrl.href);
83
120
}
84
121
} catch {}
···
126
163
await rewriter.end();
127
164
rewriter.free();
128
165
166
+
const urlPrefix = url.href.endsWith('/') ? url.href.slice(0, -1) : url.href;
167
+
const hasChildResources = [...links, ...assets].some((resource) => {
168
+
return resource.startsWith(urlPrefix + '/');
169
+
});
170
+
171
+
let filePath = url.pathname;
172
+
if (filePath.endsWith('/') || filePath === '') {
173
+
filePath = filePath + 'index.html';
174
+
} else if (hasChildResources) {
175
+
filePath = filePath + '/index.html';
176
+
} else if (!filePath.endsWith('.html') && !filePath.endsWith('.htm')) {
177
+
filePath = filePath + '.html';
178
+
}
179
+
180
+
const outputPath = join(outputDir, url.hostname, filePath);
181
+
182
+
let cached = false;
183
+
184
+
try {
185
+
await access(outputPath);
186
+
187
+
if (overwrite === 'ignore') {
188
+
const fileStats = await stat(outputPath);
189
+
logFileStats({
190
+
url: url.href,
191
+
path: outputPath,
192
+
size: fileStats.size,
193
+
links: links.size,
194
+
assets: assets.size,
195
+
cached: true,
196
+
});
197
+
return;
198
+
}
199
+
200
+
if (overwrite === 'cache') {
201
+
cached = true;
202
+
} else {
203
+
await mkdir(dirname(outputPath), { recursive: true });
204
+
await writeFile(outputPath, html);
205
+
}
206
+
} catch {
207
+
await mkdir(dirname(outputPath), { recursive: true });
208
+
await writeFile(outputPath, html);
209
+
}
210
+
129
211
for (const link of links) {
130
-
if (!queue.has(link) && !processed.has(link)) {
131
-
queue.add(link);
212
+
const linkWithoutHash = link.split('#')[0];
213
+
if (!queue.has(linkWithoutHash) && !processed.has(linkWithoutHash)) {
214
+
queue.add(linkWithoutHash);
132
215
}
133
216
}
134
217
135
218
for (const asset of assets) {
136
219
try {
137
-
const assetPath = new URL(asset);
138
-
const assetOutputPath = join(outputDir, assetPath.hostname, assetPath.pathname);
139
-
140
-
try {
141
-
await access(assetOutputPath);
142
-
143
-
if (overwrite === 'ignore') {
144
-
continue;
145
-
}
146
-
147
-
if (overwrite === 'cache') {
148
-
continue;
149
-
}
150
-
} catch {}
151
-
152
-
const assetResponse = await fetch(asset);
153
-
if (assetResponse.ok) {
154
-
const assetData = await assetResponse.arrayBuffer();
155
-
await mkdir(dirname(assetOutputPath), { recursive: true });
156
-
await writeFile(assetOutputPath, Buffer.from(assetData));
157
-
}
220
+
await downloadAsset(new URL(asset), outputDir, overwrite);
158
221
} catch {}
159
222
}
223
+
224
+
const fileStats = await stat(outputPath);
225
+
logFileStats({
226
+
url: url.href,
227
+
path: outputPath,
228
+
size: fileStats.size,
229
+
links: links.size,
230
+
assets: assets.size,
231
+
cached,
232
+
});
160
233
}
161
234
162
235
const command = define({
···
176
249
return;
177
250
}
178
251
179
-
const outputDir = output ?? getDefaultOutput(new URL(urlList[0]));
252
+
const outputDir = resolve(process.cwd(), output ?? getDefaultOutput(new URL(urlList[0])));
180
253
const linkPredicate = predicate === 'origin' ? originPredicate(new URL(urlList[0])) : subtreePredicate(new URL(urlList[0]));
181
254
182
255
const queue = new Set<string>(urlList);
···
189
262
await scrapePage(new URL(nextUrl), outputDir, linkPredicate, queue, processed, overwrite as OverwriteMode);
190
263
}
191
264
192
-
console.log(`Scraping complete. ${processed.size} pages processed.`);
265
+
console.error(`Scraping complete. ${processed.size} pages processed.`);
193
266
},
194
267
});
195
268