A simple tool which lets you scrape twitter accounts and crosspost them to bluesky accounts! Comes with a CLI and a webapp for managing profiles! Works with images/videos/link embeds/threads.
feat: migrate processed tweets tracking to SQLite for robustness
+549
-33
+391
package-lock.json
+391
package-lock.json
···
13
13
"@steipete/bird": "^0.4.0",
14
14
"axios": "^1.13.2",
15
15
"bcryptjs": "^3.0.3",
16
+
"better-sqlite3": "^12.5.0",
16
17
"commander": "^14.0.2",
17
18
"cors": "^2.8.5",
18
19
"dotenv": "^17.2.3",
···
27
28
"devDependencies": {
28
29
"@biomejs/biome": "^1.9.4",
29
30
"@types/bcryptjs": "^2.4.6",
31
+
"@types/better-sqlite3": "^7.6.13",
30
32
"@types/cors": "^2.8.19",
31
33
"@types/express": "^5.0.6",
32
34
"@types/inquirer": "^9.0.9",
···
1099
1101
"dev": true,
1100
1102
"license": "MIT"
1101
1103
},
1104
+
"node_modules/@types/better-sqlite3": {
1105
+
"version": "7.6.13",
1106
+
"resolved": "https://registry.npmjs.org/@types/better-sqlite3/-/better-sqlite3-7.6.13.tgz",
1107
+
"integrity": "sha512-NMv9ASNARoKksWtsq/SHakpYAYnhBrQgGD8zkLYk/jaK8jUGn08CfEdTRgYhMypUQAfzSP8W6gNLe0q19/t4VA==",
1108
+
"dev": true,
1109
+
"license": "MIT",
1110
+
"dependencies": {
1111
+
"@types/node": "*"
1112
+
}
1113
+
},
1102
1114
"node_modules/@types/body-parser": {
1103
1115
"version": "1.19.6",
1104
1116
"resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.6.tgz",
···
1467
1479
"bare-path": "^3.0.0"
1468
1480
}
1469
1481
},
1482
+
"node_modules/base64-js": {
1483
+
"version": "1.5.1",
1484
+
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
1485
+
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
1486
+
"funding": [
1487
+
{
1488
+
"type": "github",
1489
+
"url": "https://github.com/sponsors/feross"
1490
+
},
1491
+
{
1492
+
"type": "patreon",
1493
+
"url": "https://www.patreon.com/feross"
1494
+
},
1495
+
{
1496
+
"type": "consulting",
1497
+
"url": "https://feross.org/support"
1498
+
}
1499
+
],
1500
+
"license": "MIT"
1501
+
},
1470
1502
"node_modules/basic-ftp": {
1471
1503
"version": "5.1.0",
1472
1504
"resolved": "https://registry.npmjs.org/basic-ftp/-/basic-ftp-5.1.0.tgz",
···
1485
1517
"bcrypt": "bin/bcrypt"
1486
1518
}
1487
1519
},
1520
+
"node_modules/better-sqlite3": {
1521
+
"version": "12.5.0",
1522
+
"resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.5.0.tgz",
1523
+
"integrity": "sha512-WwCZ/5Diz7rsF29o27o0Gcc1Du+l7Zsv7SYtVPG0X3G/uUI1LqdxrQI7c9Hs2FWpqXXERjW9hp6g3/tH7DlVKg==",
1524
+
"hasInstallScript": true,
1525
+
"license": "MIT",
1526
+
"dependencies": {
1527
+
"bindings": "^1.5.0",
1528
+
"prebuild-install": "^7.1.1"
1529
+
},
1530
+
"engines": {
1531
+
"node": "20.x || 22.x || 23.x || 24.x || 25.x"
1532
+
}
1533
+
},
1534
+
"node_modules/bindings": {
1535
+
"version": "1.5.0",
1536
+
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
1537
+
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
1538
+
"license": "MIT",
1539
+
"dependencies": {
1540
+
"file-uri-to-path": "1.0.0"
1541
+
}
1542
+
},
1543
+
"node_modules/bl": {
1544
+
"version": "4.1.0",
1545
+
"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
1546
+
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
1547
+
"license": "MIT",
1548
+
"dependencies": {
1549
+
"buffer": "^5.5.0",
1550
+
"inherits": "^2.0.4",
1551
+
"readable-stream": "^3.4.0"
1552
+
}
1553
+
},
1488
1554
"node_modules/body-parser": {
1489
1555
"version": "2.2.1",
1490
1556
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.1.tgz",
···
1509
1575
"url": "https://opencollective.com/express"
1510
1576
}
1511
1577
},
1578
+
"node_modules/buffer": {
1579
+
"version": "5.7.1",
1580
+
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
1581
+
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
1582
+
"funding": [
1583
+
{
1584
+
"type": "github",
1585
+
"url": "https://github.com/sponsors/feross"
1586
+
},
1587
+
{
1588
+
"type": "patreon",
1589
+
"url": "https://www.patreon.com/feross"
1590
+
},
1591
+
{
1592
+
"type": "consulting",
1593
+
"url": "https://feross.org/support"
1594
+
}
1595
+
],
1596
+
"license": "MIT",
1597
+
"dependencies": {
1598
+
"base64-js": "^1.3.1",
1599
+
"ieee754": "^1.1.13"
1600
+
}
1601
+
},
1512
1602
"node_modules/buffer-crc32": {
1513
1603
"version": "0.2.13",
1514
1604
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
···
1567
1657
"resolved": "https://registry.npmjs.org/chardet/-/chardet-2.1.1.tgz",
1568
1658
"integrity": "sha512-PsezH1rqdV9VvyNhxxOW32/d75r01NY7TQCmOqomRo15ZSOKbpTFVsfjghxo6JloQUCGnH4k1LGu0R4yCLlWQQ==",
1569
1659
"license": "MIT"
1660
+
},
1661
+
"node_modules/chownr": {
1662
+
"version": "1.1.4",
1663
+
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
1664
+
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
1665
+
"license": "ISC"
1570
1666
},
1571
1667
"node_modules/chromium-bidi": {
1572
1668
"version": "12.0.1",
···
1805
1901
}
1806
1902
}
1807
1903
},
1904
+
"node_modules/decompress-response": {
1905
+
"version": "6.0.0",
1906
+
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
1907
+
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
1908
+
"license": "MIT",
1909
+
"dependencies": {
1910
+
"mimic-response": "^3.1.0"
1911
+
},
1912
+
"engines": {
1913
+
"node": ">=10"
1914
+
},
1915
+
"funding": {
1916
+
"url": "https://github.com/sponsors/sindresorhus"
1917
+
}
1918
+
},
1919
+
"node_modules/deep-extend": {
1920
+
"version": "0.6.0",
1921
+
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
1922
+
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
1923
+
"license": "MIT",
1924
+
"engines": {
1925
+
"node": ">=4.0.0"
1926
+
}
1927
+
},
1808
1928
"node_modules/degenerator": {
1809
1929
"version": "5.0.1",
1810
1930
"resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz",
···
1837
1957
"node": ">= 0.8"
1838
1958
}
1839
1959
},
1960
+
"node_modules/detect-libc": {
1961
+
"version": "2.1.2",
1962
+
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
1963
+
"integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",
1964
+
"license": "Apache-2.0",
1965
+
"engines": {
1966
+
"node": ">=8"
1967
+
}
1968
+
},
1840
1969
"node_modules/devtools-protocol": {
1841
1970
"version": "0.0.1534754",
1842
1971
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1534754.tgz",
···
2080
2209
"bare-events": "^2.7.0"
2081
2210
}
2082
2211
},
2212
+
"node_modules/expand-template": {
2213
+
"version": "2.0.3",
2214
+
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
2215
+
"integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
2216
+
"license": "(MIT OR WTFPL)",
2217
+
"engines": {
2218
+
"node": ">=6"
2219
+
}
2220
+
},
2083
2221
"node_modules/express": {
2084
2222
"version": "5.2.1",
2085
2223
"resolved": "https://registry.npmjs.org/express/-/express-5.2.1.tgz",
···
2183
2321
"pend": "~1.2.0"
2184
2322
}
2185
2323
},
2324
+
"node_modules/file-uri-to-path": {
2325
+
"version": "1.0.0",
2326
+
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
2327
+
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
2328
+
"license": "MIT"
2329
+
},
2186
2330
"node_modules/finalhandler": {
2187
2331
"version": "2.1.1",
2188
2332
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz",
···
2271
2415
"node": ">= 0.8"
2272
2416
}
2273
2417
},
2418
+
"node_modules/fs-constants": {
2419
+
"version": "1.0.0",
2420
+
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
2421
+
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
2422
+
"license": "MIT"
2423
+
},
2274
2424
"node_modules/fsevents": {
2275
2425
"version": "2.3.3",
2276
2426
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
···
2395
2545
"node": ">= 14"
2396
2546
}
2397
2547
},
2548
+
"node_modules/github-from-package": {
2549
+
"version": "0.0.0",
2550
+
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
2551
+
"integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
2552
+
"license": "MIT"
2553
+
},
2398
2554
"node_modules/gopd": {
2399
2555
"version": "1.2.0",
2400
2556
"resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz",
···
2508
2664
"url": "https://opencollective.com/express"
2509
2665
}
2510
2666
},
2667
+
"node_modules/ieee754": {
2668
+
"version": "1.2.1",
2669
+
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
2670
+
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
2671
+
"funding": [
2672
+
{
2673
+
"type": "github",
2674
+
"url": "https://github.com/sponsors/feross"
2675
+
},
2676
+
{
2677
+
"type": "patreon",
2678
+
"url": "https://www.patreon.com/feross"
2679
+
},
2680
+
{
2681
+
"type": "consulting",
2682
+
"url": "https://feross.org/support"
2683
+
}
2684
+
],
2685
+
"license": "BSD-3-Clause"
2686
+
},
2511
2687
"node_modules/inherits": {
2512
2688
"version": "2.0.4",
2513
2689
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
2514
2690
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
2691
+
"license": "ISC"
2692
+
},
2693
+
"node_modules/ini": {
2694
+
"version": "1.3.8",
2695
+
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
2696
+
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
2515
2697
"license": "ISC"
2516
2698
},
2517
2699
"node_modules/inquirer": {
···
2754
2936
"node": ">= 0.6"
2755
2937
}
2756
2938
},
2939
+
"node_modules/mimic-response": {
2940
+
"version": "3.1.0",
2941
+
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
2942
+
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
2943
+
"license": "MIT",
2944
+
"engines": {
2945
+
"node": ">=10"
2946
+
},
2947
+
"funding": {
2948
+
"url": "https://github.com/sponsors/sindresorhus"
2949
+
}
2950
+
},
2951
+
"node_modules/minimist": {
2952
+
"version": "1.2.8",
2953
+
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
2954
+
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
2955
+
"license": "MIT",
2956
+
"funding": {
2957
+
"url": "https://github.com/sponsors/ljharb"
2958
+
}
2959
+
},
2757
2960
"node_modules/mitt": {
2758
2961
"version": "3.0.1",
2759
2962
"resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz",
2760
2963
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==",
2964
+
"license": "MIT"
2965
+
},
2966
+
"node_modules/mkdirp-classic": {
2967
+
"version": "0.5.3",
2968
+
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
2969
+
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
2761
2970
"license": "MIT"
2762
2971
},
2763
2972
"node_modules/ms": {
···
2791
3000
"url": "https://github.com/sponsors/wooorm"
2792
3001
}
2793
3002
},
3003
+
"node_modules/napi-build-utils": {
3004
+
"version": "2.0.0",
3005
+
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
3006
+
"integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==",
3007
+
"license": "MIT"
3008
+
},
2794
3009
"node_modules/negotiator": {
2795
3010
"version": "1.0.0",
2796
3011
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-1.0.0.tgz",
···
2807
3022
"license": "MIT",
2808
3023
"engines": {
2809
3024
"node": ">= 0.4.0"
3025
+
}
3026
+
},
3027
+
"node_modules/node-abi": {
3028
+
"version": "3.85.0",
3029
+
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.85.0.tgz",
3030
+
"integrity": "sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg==",
3031
+
"license": "MIT",
3032
+
"dependencies": {
3033
+
"semver": "^7.3.5"
3034
+
},
3035
+
"engines": {
3036
+
"node": ">=10"
2810
3037
}
2811
3038
},
2812
3039
"node_modules/node-cron": {
···
2917
3144
"integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==",
2918
3145
"license": "MIT"
2919
3146
},
3147
+
"node_modules/prebuild-install": {
3148
+
"version": "7.1.3",
3149
+
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
3150
+
"integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==",
3151
+
"license": "MIT",
3152
+
"dependencies": {
3153
+
"detect-libc": "^2.0.0",
3154
+
"expand-template": "^2.0.3",
3155
+
"github-from-package": "0.0.0",
3156
+
"minimist": "^1.2.3",
3157
+
"mkdirp-classic": "^0.5.3",
3158
+
"napi-build-utils": "^2.0.0",
3159
+
"node-abi": "^3.3.0",
3160
+
"pump": "^3.0.0",
3161
+
"rc": "^1.2.7",
3162
+
"simple-get": "^4.0.0",
3163
+
"tar-fs": "^2.0.0",
3164
+
"tunnel-agent": "^0.6.0"
3165
+
},
3166
+
"bin": {
3167
+
"prebuild-install": "bin.js"
3168
+
},
3169
+
"engines": {
3170
+
"node": ">=10"
3171
+
}
3172
+
},
3173
+
"node_modules/prebuild-install/node_modules/tar-fs": {
3174
+
"version": "2.1.4",
3175
+
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
3176
+
"integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
3177
+
"license": "MIT",
3178
+
"dependencies": {
3179
+
"chownr": "^1.1.1",
3180
+
"mkdirp-classic": "^0.5.2",
3181
+
"pump": "^3.0.0",
3182
+
"tar-stream": "^2.1.4"
3183
+
}
3184
+
},
3185
+
"node_modules/prebuild-install/node_modules/tar-stream": {
3186
+
"version": "2.2.0",
3187
+
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
3188
+
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
3189
+
"license": "MIT",
3190
+
"dependencies": {
3191
+
"bl": "^4.0.3",
3192
+
"end-of-stream": "^1.4.1",
3193
+
"fs-constants": "^1.0.0",
3194
+
"inherits": "^2.0.3",
3195
+
"readable-stream": "^3.1.1"
3196
+
},
3197
+
"engines": {
3198
+
"node": ">=6"
3199
+
}
3200
+
},
2920
3201
"node_modules/progress": {
2921
3202
"version": "2.0.3",
2922
3203
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
···
3031
3312
"node": ">= 0.10"
3032
3313
}
3033
3314
},
3315
+
"node_modules/rc": {
3316
+
"version": "1.2.8",
3317
+
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
3318
+
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
3319
+
"license": "(BSD-2-Clause OR MIT OR Apache-2.0)",
3320
+
"dependencies": {
3321
+
"deep-extend": "^0.6.0",
3322
+
"ini": "~1.3.0",
3323
+
"minimist": "^1.2.0",
3324
+
"strip-json-comments": "~2.0.1"
3325
+
},
3326
+
"bin": {
3327
+
"rc": "cli.js"
3328
+
}
3329
+
},
3330
+
"node_modules/readable-stream": {
3331
+
"version": "3.6.2",
3332
+
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
3333
+
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
3334
+
"license": "MIT",
3335
+
"dependencies": {
3336
+
"inherits": "^2.0.3",
3337
+
"string_decoder": "^1.1.1",
3338
+
"util-deprecate": "^1.0.1"
3339
+
},
3340
+
"engines": {
3341
+
"node": ">= 6"
3342
+
}
3343
+
},
3034
3344
"node_modules/require-directory": {
3035
3345
"version": "2.1.1",
3036
3346
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
···
3282
3592
"url": "https://github.com/sponsors/isaacs"
3283
3593
}
3284
3594
},
3595
+
"node_modules/simple-concat": {
3596
+
"version": "1.0.1",
3597
+
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
3598
+
"integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
3599
+
"funding": [
3600
+
{
3601
+
"type": "github",
3602
+
"url": "https://github.com/sponsors/feross"
3603
+
},
3604
+
{
3605
+
"type": "patreon",
3606
+
"url": "https://www.patreon.com/feross"
3607
+
},
3608
+
{
3609
+
"type": "consulting",
3610
+
"url": "https://feross.org/support"
3611
+
}
3612
+
],
3613
+
"license": "MIT"
3614
+
},
3615
+
"node_modules/simple-get": {
3616
+
"version": "4.0.1",
3617
+
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
3618
+
"integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
3619
+
"funding": [
3620
+
{
3621
+
"type": "github",
3622
+
"url": "https://github.com/sponsors/feross"
3623
+
},
3624
+
{
3625
+
"type": "patreon",
3626
+
"url": "https://www.patreon.com/feross"
3627
+
},
3628
+
{
3629
+
"type": "consulting",
3630
+
"url": "https://feross.org/support"
3631
+
}
3632
+
],
3633
+
"license": "MIT",
3634
+
"dependencies": {
3635
+
"decompress-response": "^6.0.0",
3636
+
"once": "^1.3.1",
3637
+
"simple-concat": "^1.0.0"
3638
+
}
3639
+
},
3285
3640
"node_modules/smart-buffer": {
3286
3641
"version": "4.2.0",
3287
3642
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
···
3350
3705
"text-decoder": "^1.1.0"
3351
3706
}
3352
3707
},
3708
+
"node_modules/string_decoder": {
3709
+
"version": "1.3.0",
3710
+
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
3711
+
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
3712
+
"license": "MIT",
3713
+
"dependencies": {
3714
+
"safe-buffer": "~5.2.0"
3715
+
}
3716
+
},
3353
3717
"node_modules/string-width": {
3354
3718
"version": "7.2.0",
3355
3719
"resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz",
···
3382
3746
"url": "https://github.com/chalk/strip-ansi?sponsor=1"
3383
3747
}
3384
3748
},
3749
+
"node_modules/strip-json-comments": {
3750
+
"version": "2.0.1",
3751
+
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
3752
+
"integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
3753
+
"license": "MIT",
3754
+
"engines": {
3755
+
"node": ">=0.10.0"
3756
+
}
3757
+
},
3385
3758
"node_modules/tar-fs": {
3386
3759
"version": "3.1.1",
3387
3760
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.1.1.tgz",
···
3474
3847
"fsevents": "~2.3.3"
3475
3848
}
3476
3849
},
3850
+
"node_modules/tunnel-agent": {
3851
+
"version": "0.6.0",
3852
+
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
3853
+
"integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
3854
+
"license": "Apache-2.0",
3855
+
"dependencies": {
3856
+
"safe-buffer": "^5.0.1"
3857
+
},
3858
+
"engines": {
3859
+
"node": "*"
3860
+
}
3861
+
},
3477
3862
"node_modules/type-is": {
3478
3863
"version": "2.0.1",
3479
3864
"resolved": "https://registry.npmjs.org/type-is/-/type-is-2.0.1.tgz",
···
3563
3948
"engines": {
3564
3949
"node": ">= 0.8"
3565
3950
}
3951
+
},
3952
+
"node_modules/util-deprecate": {
3953
+
"version": "1.0.2",
3954
+
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
3955
+
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
3956
+
"license": "MIT"
3566
3957
},
3567
3958
"node_modules/vary": {
3568
3959
"version": "1.1.2",
+2
package.json
+2
package.json
···
29
29
"@steipete/bird": "^0.4.0",
30
30
"axios": "^1.13.2",
31
31
"bcryptjs": "^3.0.3",
32
+
"better-sqlite3": "^12.5.0",
32
33
"commander": "^14.0.2",
33
34
"cors": "^2.8.5",
34
35
"dotenv": "^17.2.3",
···
43
44
"devDependencies": {
44
45
"@biomejs/biome": "^1.9.4",
45
46
"@types/bcryptjs": "^2.4.6",
47
+
"@types/better-sqlite3": "^7.6.13",
46
48
"@types/cors": "^2.8.19",
47
49
"@types/express": "^5.0.6",
48
50
"@types/inquirer": "^9.0.9",
+101
src/db.ts
+101
src/db.ts
···
1
+
import Database from 'better-sqlite3';
2
+
import path from 'node:path';
3
+
import fs from 'node:fs';
4
+
import { fileURLToPath } from 'node:url';
5
+
6
+
const __filename = fileURLToPath(import.meta.url);
7
+
const __dirname = path.dirname(__filename);
8
+
9
+
const DB_DIR = path.join(__dirname, '..', 'data');
10
+
if (!fs.existsSync(DB_DIR)) {
11
+
fs.mkdirSync(DB_DIR);
12
+
}
13
+
14
+
const db = new Database(path.join(DB_DIR, 'database.sqlite'));
15
+
16
+
// Enable WAL mode for better concurrency
17
+
db.pragma('journal_mode = WAL');
18
+
19
+
// Initialize schema
20
+
db.exec(`
21
+
CREATE TABLE IF NOT EXISTS processed_tweets (
22
+
twitter_id TEXT PRIMARY KEY,
23
+
twitter_username TEXT NOT NULL,
24
+
bsky_uri TEXT,
25
+
bsky_cid TEXT,
26
+
bsky_root_uri TEXT,
27
+
bsky_root_cid TEXT,
28
+
status TEXT NOT NULL, -- 'migrated', 'skipped', 'failed'
29
+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
30
+
);
31
+
CREATE INDEX IF NOT EXISTS idx_twitter_username ON processed_tweets(twitter_username);
32
+
`);
33
+
34
+
export interface ProcessedTweet {
35
+
twitter_id: string;
36
+
twitter_username: string;
37
+
bsky_uri?: string;
38
+
bsky_cid?: string;
39
+
bsky_root_uri?: string;
40
+
bsky_root_cid?: string;
41
+
status: 'migrated' | 'skipped' | 'failed';
42
+
}
43
+
44
+
export const dbService = {
45
+
getTweet(twitterId: string): ProcessedTweet | null {
46
+
const stmt = db.prepare('SELECT * FROM processed_tweets WHERE twitter_id = ?');
47
+
const row = stmt.get(twitterId) as any;
48
+
if (!row) return null;
49
+
return {
50
+
twitter_id: row.twitter_id,
51
+
twitter_username: row.twitter_username,
52
+
bsky_uri: row.bsky_uri,
53
+
bsky_cid: row.bsky_cid,
54
+
bsky_root_uri: row.bsky_root_uri,
55
+
bsky_root_cid: row.bsky_root_cid,
56
+
status: row.status
57
+
};
58
+
},
59
+
60
+
saveTweet(tweet: ProcessedTweet) {
61
+
const stmt = db.prepare(`
62
+
INSERT OR REPLACE INTO processed_tweets
63
+
(twitter_id, twitter_username, bsky_uri, bsky_cid, bsky_root_uri, bsky_root_cid, status)
64
+
VALUES (?, ?, ?, ?, ?, ?, ?)
65
+
`);
66
+
stmt.run(
67
+
tweet.twitter_id,
68
+
tweet.twitter_username,
69
+
tweet.bsky_uri || null,
70
+
tweet.bsky_cid || null,
71
+
tweet.bsky_root_uri || null,
72
+
tweet.bsky_root_cid || null,
73
+
tweet.status
74
+
);
75
+
},
76
+
77
+
getTweetsByUsername(username: string): Record<string, any> {
78
+
const stmt = db.prepare('SELECT * FROM processed_tweets WHERE twitter_username = ?');
79
+
const rows = stmt.all(username.toLowerCase()) as any[];
80
+
const map: Record<string, any> = {};
81
+
for (const row of rows) {
82
+
map[row.twitter_id] = {
83
+
uri: row.bsky_uri,
84
+
cid: row.bsky_cid,
85
+
root: row.bsky_root_uri ? { uri: row.bsky_root_uri, cid: row.bsky_root_cid } : undefined,
86
+
migrated: row.status === 'migrated',
87
+
skipped: row.status === 'skipped'
88
+
};
89
+
}
90
+
return map;
91
+
},
92
+
93
+
deleteTweetsByUsername(username: string) {
94
+
const stmt = db.prepare('DELETE FROM processed_tweets WHERE twitter_username = ?');
95
+
stmt.run(username.toLowerCase());
96
+
},
97
+
98
+
clearAll() {
99
+
db.prepare('DELETE FROM processed_tweets').run();
100
+
}
101
+
};
+51
-25
src/index.ts
+51
-25
src/index.ts
···
112
112
aspectRatio?: AspectRatio;
113
113
}
114
114
115
+
import { dbService } from './db.js';
116
+
115
117
// ============================================================================
116
118
// State Management
117
119
// ============================================================================
118
120
119
121
const PROCESSED_DIR = path.join(__dirname, '..', 'processed');
120
-
if (!fs.existsSync(PROCESSED_DIR)) {
121
-
fs.mkdirSync(PROCESSED_DIR);
122
-
}
123
122
124
-
function getProcessedFilePath(twitterUsername: string): string {
125
-
return path.join(PROCESSED_DIR, `${twitterUsername.toLowerCase()}.json`);
126
-
}
123
+
async function migrateJsonToSqlite() {
124
+
if (!fs.existsSync(PROCESSED_DIR)) return;
125
+
126
+
const files = fs.readdirSync(PROCESSED_DIR).filter(f => f.endsWith('.json'));
127
+
if (files.length === 0) return;
127
128
128
-
function loadProcessedTweets(twitterUsername: string): ProcessedTweetsMap {
129
-
const filePath = getProcessedFilePath(twitterUsername);
130
-
try {
131
-
if (fs.existsSync(filePath)) {
132
-
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
129
+
console.log(`📦 Found ${files.length} legacy cache files. Migrating to SQLite...`);
130
+
131
+
for (const file of files) {
132
+
const username = file.replace('.json', '');
133
+
try {
134
+
const filePath = path.join(PROCESSED_DIR, file);
135
+
const data = JSON.parse(fs.readFileSync(filePath, 'utf8')) as ProcessedTweetsMap;
136
+
137
+
for (const [twitterId, entry] of Object.entries(data)) {
138
+
dbService.saveTweet({
139
+
twitter_id: twitterId,
140
+
twitter_username: username,
141
+
bsky_uri: entry.uri,
142
+
bsky_cid: entry.cid,
143
+
bsky_root_uri: entry.root?.uri,
144
+
bsky_root_cid: entry.root?.cid,
145
+
status: entry.migrated ? 'migrated' : (entry.skipped ? 'skipped' : 'failed')
146
+
});
147
+
}
148
+
// Move file to backup
149
+
const backupDir = path.join(PROCESSED_DIR, 'backup');
150
+
if (!fs.existsSync(backupDir)) fs.mkdirSync(backupDir);
151
+
fs.renameSync(filePath, path.join(backupDir, file));
152
+
} catch (err) {
153
+
console.error(`❌ Failed to migrate ${file}:`, err);
133
154
}
134
-
} catch (err) {
135
-
console.error(`Error loading processed tweets for ${twitterUsername}:`, err);
136
155
}
137
-
return {};
156
+
console.log('✅ Migration complete.');
138
157
}
139
158
140
-
function saveProcessedTweets(twitterUsername: string, data: ProcessedTweetsMap): void {
141
-
const filePath = getProcessedFilePath(twitterUsername);
142
-
try {
143
-
fs.writeFileSync(filePath, JSON.stringify(data, null, 2));
144
-
} catch (err) {
145
-
console.error(`Error saving processed tweets for ${twitterUsername}:`, err);
146
-
}
159
+
function loadProcessedTweets(twitterUsername: string): ProcessedTweetsMap {
160
+
return dbService.getTweetsByUsername(twitterUsername);
161
+
}
162
+
163
+
function saveProcessedTweet(twitterUsername: string, twitterId: string, entry: ProcessedTweetEntry): void {
164
+
dbService.saveTweet({
165
+
twitter_id: twitterId,
166
+
twitter_username: twitterUsername.toLowerCase(),
167
+
bsky_uri: entry.uri,
168
+
bsky_cid: entry.cid,
169
+
bsky_root_uri: entry.root?.uri,
170
+
bsky_root_cid: entry.root?.cid,
171
+
status: entry.migrated || (entry.uri && entry.cid) ? 'migrated' : (entry.skipped ? 'skipped' : 'failed')
172
+
});
147
173
}
148
174
149
175
// ============================================================================
···
592
618
} else {
593
619
console.log(`[${twitterUsername}] ⏩ Skipping external/unknown reply.`);
594
620
if (!dryRun) {
595
-
processedTweets[tweetId] = { skipped: true };
596
-
saveProcessedTweets(twitterUsername, processedTweets);
621
+
saveProcessedTweet(twitterUsername, tweetId, { skipped: true });
597
622
}
598
623
continue;
599
624
}
···
810
835
};
811
836
812
837
if (i === 0) {
813
-
processedTweets[tweetId] = currentPostInfo;
814
-
saveProcessedTweets(twitterUsername, processedTweets);
838
+
saveProcessedTweet(twitterUsername, tweetId, currentPostInfo);
815
839
}
816
840
817
841
lastPostInfo = currentPostInfo;
···
984
1008
const options = program.opts();
985
1009
986
1010
const config = getConfig();
1011
+
1012
+
await migrateJsonToSqlite();
987
1013
988
1014
if (!options.web) {
989
1015
console.log('🌐 Web interface is disabled.');
+4
-8
src/server.ts
+4
-8
src/server.ts
···
6
6
import express from 'express';
7
7
import jwt from 'jsonwebtoken';
8
8
import { getConfig, saveConfig } from './config-manager.js';
9
+
import { dbService } from './db.js';
9
10
10
11
const __filename = fileURLToPath(import.meta.url);
11
12
const __dirname = path.dirname(__filename);
···
145
146
return;
146
147
}
147
148
148
-
const cachePath = path.join(__dirname, '../processed', `${mapping.twitterUsername.toLowerCase()}.json`);
149
-
if (fs.existsSync(cachePath)) {
150
-
fs.unlinkSync(cachePath);
151
-
res.json({ success: true, message: 'Cache cleared' });
152
-
} else {
153
-
res.json({ success: true, message: 'No cache found' });
154
-
}
149
+
dbService.deleteTweetsByUsername(mapping.twitterUsername);
150
+
res.json({ success: true, message: 'Cache cleared' });
155
151
});
156
152
157
153
// --- Twitter Config Routes (Admin Only) ---
···
261
257
console.log(`🚀 Web interface running at http://localhost:${PORT}`);
262
258
console.log('📡 Accessible on your local network/Tailscale via your IP.');
263
259
});
264
-
}
260
+
}