Import your Last.fm and Spotify listening history to the AT Protocol network using the
fm.teal.alpha.feed.play lexicon.
feat: add duplicate removal CLI and update publisher import
Add --remove-duplicates flag and consolidate publisher imports.
Changes:
- Update version string to v0.2.0
- Change import from './publisher-applywrites.js' to './publisher.js'
- Add removeDuplicates import from sync.js
- Add --remove-duplicates CLI flag to help text
- Update default batch-delay in help: 2000ms → 500ms
- Add 'remove-duplicates' to parseCommandLineArgs
- Add removeDuplicatesMode handling logic
+43
-4
+43
-4
src/lib/cli.ts
+43
-4
src/lib/cli.ts
···
3
3
import type { PlayRecord, Config, CommandLineArgs, PublishResult } from '../types.js';
4
4
import { login } from './auth.js';
5
5
import { parseLastFmCsv, convertToPlayRecord, sortRecords } from '../lib/csv.js';
6
-
import { publishRecordsWithApplyWrites } from './publisher-applywrites.js';
6
+
import { publishRecordsWithApplyWrites } from './publisher.js';
7
7
import { prompt } from '../utils/input.js';
8
8
import config from '../config.js';
9
9
import { calculateOptimalBatchSize, showRateLimitInfo } from '../utils/helpers.js';
10
-
import { fetchExistingRecords, filterNewRecords, displaySyncStats } from './sync.js';
10
+
import { fetchExistingRecords, filterNewRecords, displaySyncStats, removeDuplicates } from './sync.js';
11
11
12
12
/**
13
13
* Show help message
14
14
*/
15
15
export function showHelp(): void {
16
16
console.log(`
17
-
Last.fm to ATProto Importer v0.1.0
17
+
Last.fm to ATProto Importer v0.2.0
18
18
19
19
Usage: npm start [options]
20
20
···
24
24
-i, --identifier <id> ATProto handle or DID
25
25
-p, --password <pass> ATProto app password
26
26
-b, --batch-size <num> Number of records per batch (auto-calculated if not set)
27
-
-d, --batch-delay <ms> Delay between batches in ms (default: 2000, min: 1000)
27
+
-d, --batch-delay <ms> Delay between batches in ms (default: 500, min: 500)
28
28
-y, --yes Skip confirmation prompt
29
29
-n, --dry-run Preview records without publishing
30
30
-r, --reverse-chronological Process newest first (default: oldest first)
31
31
-s, --sync Re-sync mode: check existing Teal records and only import new ones
32
+
--remove-duplicates Remove duplicate records from Teal (keeps first occurrence)
32
33
`);
33
34
}
34
35
···
48
49
'dry-run': { type: 'boolean', short: 'n', default: false },
49
50
'reverse-chronological': { type: 'boolean', short: 'r', default: false },
50
51
sync: { type: 'boolean', short: 's', default: false },
52
+
'remove-duplicates': { type: 'boolean', default: false },
51
53
} as const;
52
54
53
55
try {
···
80
82
81
83
const dryRun = args['dry-run'] ?? false;
82
84
const syncMode = args.sync ?? false;
85
+
const removeDuplicatesMode = args['remove-duplicates'] ?? false;
83
86
let agent: AtpAgent | null = null;
87
+
88
+
// Remove duplicates mode - requires authentication but not file
89
+
if (removeDuplicatesMode) {
90
+
if (!args.identifier || !args.password) {
91
+
throw new Error('Missing required arguments for login: -i (identifier) and -p (password)');
92
+
}
93
+
94
+
agent = await login(args.identifier, args.password, cfg.SLINGSHOT_RESOLVER) as AtpAgent;
95
+
96
+
// Check for duplicates first
97
+
const result = await removeDuplicates(agent, cfg, true); // Always dry-run first to show info
98
+
99
+
if (result.totalDuplicates === 0) {
100
+
return; // No duplicates, exit early
101
+
}
102
+
103
+
// Ask for confirmation if not in dry-run mode
104
+
if (!dryRun && !(args.yes ?? false)) {
105
+
console.log(`⚠️ WARNING: This will permanently delete ${result.totalDuplicates} duplicate records from Teal.`);
106
+
console.log(' The first occurrence of each duplicate will be kept.\n');
107
+
const answer = await prompt('Are you sure you want to continue? (y/N) ');
108
+
if (answer.toLowerCase() !== 'y') {
109
+
console.log('Duplicate removal cancelled by user.');
110
+
process.exit(0);
111
+
}
112
+
113
+
// Actually remove duplicates
114
+
await removeDuplicates(agent, cfg, false);
115
+
console.log('🎉 Duplicate removal complete!\n');
116
+
} else if (dryRun) {
117
+
console.log('DRY RUN: No records were actually removed.\n');
118
+
console.log('Remove --dry-run flag to actually delete duplicates.\n');
119
+
}
120
+
121
+
return;
122
+
}
84
123
85
124
// 1. Get Authentication (required for sync mode, even in dry-run)
86
125
if (!dryRun || syncMode) {