j4ck.xyz / tweets2bsky
A simple tool which lets you scrape twitter accounts and crosspost them to bluesky accounts! Comes with a CLI and a webapp for managing profiles! Works with images/videos/link embeds/threads.
11
fork

Configure Feed

Select the types of activity you want to include in your feed.

Fix bio t.co expansion and quote-safe mirrored descriptions

jack 38f6cd2e 8e9bba2c

+69 -19
+69 -19
src/profile-mirror.ts
··· 36 36 's', 37 37 'si', 38 38 ]); 39 + const URL_EXPANSION_HEADERS = { 40 + 'user-agent': 41 + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', 42 + accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 43 + }; 39 44 40 45 type ProfileImageKind = 'avatar' | 'banner'; 41 46 ··· 274 279 return normalizeOptionalString(res?.responseUrl); 275 280 }; 276 281 282 + const decodeEscapedUrlValue = (value: string): string => { 283 + return value 284 + .replace(/\\\//g, '/') 285 + .replace(/&amp;/gi, '&') 286 + .replace(/&quot;/gi, '"') 287 + .trim(); 288 + }; 289 + 290 + const extractRedirectUrlFromHtml = (html: string): string | undefined => { 291 + const metaRefresh = 292 + html.match(/http-equiv=["']refresh["'][^>]*content=["'][^"']*url=([^"'>\s]+)/i)?.[1] || 293 + html.match(/<meta[^>]+content=["'][^"']*url=([^"'>\s]+)/i)?.[1]; 294 + if (metaRefresh) { 295 + return decodeEscapedUrlValue(metaRefresh); 296 + } 297 + 298 + const locationReplace = 299 + html.match(/location\.replace\(\s*(["'])(.+?)\1\s*\)/i)?.[2] || 300 + html.match(/window\.location(?:\.href)?\s*=\s*(["'])(.+?)\1/i)?.[2]; 301 + if (locationReplace) { 302 + return decodeEscapedUrlValue(locationReplace); 303 + } 304 + 305 + const titleUrl = html.match(/<title>\s*(https?:\/\/[^<\s]+)\s*<\/title>/i)?.[1]; 306 + if (titleUrl) { 307 + return decodeEscapedUrlValue(titleUrl); 308 + } 309 + 310 + return undefined; 311 + }; 312 + 277 313 const expandShortUrl = async (shortUrl: string): Promise<string> => { 278 314 try { 279 315 const head = await axios.head(shortUrl, { 280 316 maxRedirects: 8, 281 317 timeout: 8_000, 318 + headers: URL_EXPANSION_HEADERS, 282 319 validateStatus: (status) => status >= 200 && status < 400, 283 320 }); 284 - return resolveRedirectUrl(head) || shortUrl; 321 + const resolvedByHead = resolveRedirectUrl(head); 322 + if (resolvedByHead && resolvedByHead !== shortUrl) { 323 + return resolvedByHead; 324 + } 285 325 } catch { 286 - try { 287 - const get = await axios.get(shortUrl, { 288 - maxRedirects: 8, 289 - timeout: 8_000, 290 - responseType: 'stream', 291 - validateStatus: (status) => status >= 200 && status < 400, 292 - }); 293 - try { 294 - get.data?.destroy?.(); 295 - } catch { 296 - // Ignore stream cleanup errors. 297 - } 298 - return resolveRedirectUrl(get) || shortUrl; 299 - } catch { 300 - return shortUrl; 326 + // Fall through to GET-based resolver. 327 + } 328 + 329 + try { 330 + const get = await axios.get<string>(shortUrl, { 331 + maxRedirects: 8, 332 + timeout: 8_000, 333 + headers: URL_EXPANSION_HEADERS, 334 + maxContentLength: 512 * 1024, 335 + validateStatus: (status) => status >= 200 && status < 400, 336 + }); 337 + 338 + const resolvedByGet = resolveRedirectUrl(get); 339 + if (resolvedByGet && resolvedByGet !== shortUrl) { 340 + return resolvedByGet; 301 341 } 342 + 343 + const html = typeof get.data === 'string' ? get.data : ''; 344 + const resolvedFromHtml = extractRedirectUrlFromHtml(html); 345 + if (resolvedFromHtml) { 346 + return resolvedFromHtml; 347 + } 348 + } catch { 349 + return shortUrl; 302 350 } 351 + 352 + return shortUrl; 303 353 }; 304 354 305 355 const expandAndNormalizeTwitterBioLinks = async (biography?: string): Promise<string | undefined> => { ··· 497 547 return truncateGraphemes(intro, 256); 498 548 } 499 549 500 - const full = `${intro}\n\n"${bio}"`; 550 + const full = `${intro}\n\n${bio}`; 501 551 if (getGraphemeSegments(full).length <= 256) { 502 552 return full; 503 553 } 504 554 505 - const reserved = getGraphemeSegments(`${intro}\n\n""`).length; 555 + const reserved = getGraphemeSegments(`${intro}\n\n`).length; 506 556 const maxBioLength = Math.max(0, 256 - reserved); 507 557 const truncatedBio = truncateGraphemes(bio, maxBioLength); 508 - return `${intro}\n\n"${truncatedBio}"`; 558 + return `${intro}\n\n${truncatedBio}`; 509 559 }; 510 560 511 561 export const fetchTwitterMirrorProfile = async (inputUsername: string): Promise<TwitterMirrorProfile> => {