jauntywk.bsky.social / dexport
Small wget like mirroring utility.
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

Fix function parameter naming conflict in replace-host feature

rektide a9491524 59c735c2

+47 -17
+47 -17
src/cli.ts
··· 39 39 return parts.length > 0 ? parts[parts.length - 1] : url.hostname; 40 40 } 41 41 42 - function replaceHost(url: URL, newHost: string): URL { 42 + function applyReplaceHost(url: URL, newHost: string): URL { 43 43 const newUrl = new URL(url.href); 44 44 newUrl.hostname = newHost; 45 45 return newUrl; ··· 54 54 outputDir: string, 55 55 overwrite: OverwriteMode, 56 56 stripHost: boolean, 57 - replaceHost?: string, 57 + replacementHost?: string, 58 58 ): Promise<boolean> { 59 59 const assetOutputPath = stripHost 60 60 ? join(outputDir, url.pathname) ··· 90 90 } 91 91 } catch {} 92 92 93 - const fetchUrl = replaceHost ? replaceHost(url, replaceHost) : url; 93 + const fetchUrl = replacementHost ? applyReplaceHost(url, replacementHost) : url; 94 94 const response = await fetch(fetchUrl.href); 95 95 if (!response.ok) { 96 96 return false; ··· 120 120 processed: Set<string>, 121 121 overwrite: OverwriteMode, 122 122 stripHost: boolean, 123 - replaceHost?: string, 123 + replacementHost?: string, 124 124 ): Promise<void> { 125 125 const urlWithoutHash = url.href.split("#")[0]; 126 126 if (processed.has(urlWithoutHash)) return; ··· 146 146 147 147 try { 148 148 let linkUrl = new URL(href, url); 149 - if (replaceHost) { 150 - linkUrl = replaceHost(linkUrl, replaceHost); 149 + if (replacementHost) { 150 + linkUrl = applyReplaceHost(linkUrl, replacementHost); 151 151 } 152 152 if (predicate(linkUrl, url) && !processed.has(linkUrl.href.split("#")[0])) { 153 153 links.add(linkUrl.href); ··· 163 163 164 164 try { 165 165 let assetUrl = new URL(src, url); 166 - if (replaceHost) { 167 - assetUrl = replaceHost(assetUrl, replaceHost); 166 + if (replacementHost) { 167 + assetUrl = applyReplaceHost(assetUrl, replacementHost); 168 168 } 169 169 assets.add(assetUrl.href); 170 170 } catch {} ··· 178 178 179 179 try { 180 180 let assetUrl = new URL(href, url); 181 - if (replaceHost) { 182 - assetUrl = replaceHost(assetUrl, replaceHost); 181 + if (replacementHost) { 182 + assetUrl = applyReplaceHost(assetUrl, replacementHost); 183 + } 184 + assets.add(assetUrl.href); 185 + } catch {} 186 + }, 187 + }); 188 + 189 + rewriter.on("script[src]", { 190 + element(element) { 191 + const src = element.getAttribute("src"); 192 + if (!src) return; 193 + 194 + try { 195 + let assetUrl = new URL(src, url); 196 + if (replacementHost) { 197 + assetUrl = applyReplaceHost(assetUrl, replacementHost); 198 + } 199 + assets.add(assetUrl.href); 200 + } catch {} 201 + }, 202 + }); 203 + 204 + rewriter.on("link[href]", { 205 + element(element) { 206 + const href = element.getAttribute("href"); 207 + if (!href) return; 208 + 209 + try { 210 + let assetUrl = new URL(src, url); 211 + if (replacementHost) { 212 + assetUrl = applyReplaceHost(assetUrl, replacementHost); 183 213 } 184 214 assets.add(assetUrl.href); 185 215 } catch {} ··· 262 292 263 293 for (const asset of assets) { 264 294 try { 265 - await downloadAsset(new URL(asset), outputDir, overwrite, stripHost, replaceHost); 295 + await downloadAsset(new URL(asset), outputDir, overwrite, stripHost, replacementHost); 266 296 } catch {} 267 297 } 268 298 ··· 296 326 "replace-host": { type: "string", description: "Replace hostname in URLs with specified hostname" }, 297 327 }, 298 328 async run(ctx) { 299 - const { url, urls, output, predicate, overwrite, "strip-host": stripHost, "replace-host": replaceHost } = ctx.values; 329 + const { url, urls, output, predicate, overwrite, "strip-host": stripHost, "replace-host": replacementHost } = ctx.values; 300 330 const urlList = url ? [url] : (urls ?? []); 301 331 302 332 if (urlList.length === 0) { ··· 307 337 const outputDir = resolve(process.cwd(), output ?? "."); 308 338 const parsedUrls = urlList.map((u) => ensureUrl(u)); 309 339 310 - if (replaceHost) { 340 + if (replacementHost) { 311 341 for (let i = 0; i < parsedUrls.length; i++) { 312 - parsedUrls[i] = replaceHost(parsedUrls[i], replaceHost); 342 + parsedUrls[i] = applyReplaceHost(parsedUrls[i], replacementHost); 313 343 } 314 344 } 315 345 ··· 326 356 queue.delete(nextUrl); 327 357 328 358 let urlObj = ensureUrl(nextUrl); 329 - if (replaceHost) { 330 - urlObj = replaceHost(urlObj, replaceHost); 359 + if (replacementHost) { 360 + urlObj = applyReplaceHost(urlObj, replacementHost); 331 361 } 332 362 333 363 await scrapePage( ··· 338 368 processed, 339 369 overwrite as OverwriteMode, 340 370 stripHost, 341 - replaceHost, 371 + replacementHost, 342 372 ); 343 373 } 344 374