Small
wget like mirroring utility.
Add positional arguments support for URLs, keep --url flag
+22
-7
+22
-7
src/cli.ts
+22
-7
src/cli.ts
···
16
16
return (url) => url.href.startsWith(baseUrl.href);
17
17
}
18
18
19
+
function getDefaultOutput(url: URL): string {
20
+
const pathname = url.pathname;
21
+
const cleanedPath = pathname.endsWith('/') ? pathname.slice(0, -1) : pathname;
22
+
const parts = cleanedPath.split('/').filter(Boolean);
23
+
return parts.length > 0 ? parts[parts.length - 1] : url.hostname;
24
+
}
25
+
19
26
async function scrapePage(url: URL, outputDir: string, predicate: LinkPredicate, queue: Set<string>, processed: Set<string>, overwrite: OverwriteMode) {
20
27
if (processed.has(url.href)) return;
21
28
···
154
161
155
162
const command = define({
156
163
args: {
157
-
url: { type: 'string', description: 'URL to start scraping', required: true },
158
-
output: { type: 'string', description: 'Output directory', required: true },
164
+
urls: { type: 'positional', description: 'URLs to scrape', multiple: true },
165
+
url: { type: 'string', description: 'URL to start scraping' },
166
+
output: { type: 'string', description: 'Output directory' },
159
167
predicate: { type: 'string', description: 'Link predicate: origin or subtree', default: 'subtree' },
160
168
overwrite: { type: 'string', description: 'Overwrite mode: ignore, cache, or update', default: 'cache' },
161
169
},
162
170
async run(ctx) {
163
-
const { url, output, predicate, overwrite } = ctx.values;
164
-
const baseUrl = new URL(url);
165
-
const linkPredicate = predicate === 'origin' ? originPredicate(baseUrl) : subtreePredicate(baseUrl);
171
+
const { url, urls, output, predicate, overwrite } = ctx.values;
172
+
const urlList = url ? [url] : (urls ?? []);
166
173
167
-
const queue = new Set<string>([baseUrl.href]);
174
+
if (urlList.length === 0) {
175
+
console.error('No URLs provided. Use positional arguments or --url.');
176
+
return;
177
+
}
178
+
179
+
const outputDir = output ?? getDefaultOutput(new URL(urlList[0]));
180
+
const linkPredicate = predicate === 'origin' ? originPredicate(new URL(urlList[0])) : subtreePredicate(new URL(urlList[0]));
181
+
182
+
const queue = new Set<string>(urlList);
168
183
const processed = new Set<string>();
169
184
170
185
while (queue.size > 0) {
171
186
const nextUrl = Array.from(queue)[0];
172
187
queue.delete(nextUrl);
173
188
174
-
await scrapePage(new URL(nextUrl), output, linkPredicate, queue, processed, overwrite as OverwriteMode);
189
+
await scrapePage(new URL(nextUrl), outputDir, linkPredicate, queue, processed, overwrite as OverwriteMode);
175
190
}
176
191
177
192
console.log(`Scraping complete. ${processed.size} pages processed.`);