Import your Last.fm and Spotify listening history to the AT Protocol network using the
fm.teal.alpha.feed.play lexicon.
feat: implement duplicate detection and locale-aware formatting
Add complete duplicate detection system with formatted output.
New functions:
- fetchAllRecords(): Fetch all records including duplicates with progress
- findDuplicates(): Group records by key and identify duplicates
- removeDuplicates(): Remove duplicate records with progress tracking
Improvements:
- Import formatDate and formatDateRange from helpers
- Update displaySyncStats() to use formatDateRange()
- Update filterNewRecords() to use formatDate() with timestamps
- Progress updates every 500 records during fetch
- Dry-run support in removeDuplicates()
- Example display (up to 5) before removal
- Progress bar with speed tracking during deletion
+189
-4
+189
-4
src/lib/sync.ts
+189
-4
src/lib/sync.ts
···
1
1
import type { AtpAgent } from '@atproto/api';
2
2
import type { PlayRecord, Config } from '../types.js';
3
+
import { formatDate, formatDateRange } from '../utils/helpers.js';
4
+
import * as ui from '../utils/ui.js';
3
5
4
6
interface ExistingRecord {
5
7
uri: string;
6
8
cid: string;
7
9
value: PlayRecord;
10
+
}
11
+
12
+
interface DuplicateGroup {
13
+
key: string;
14
+
records: ExistingRecord[];
8
15
}
9
16
10
17
/**
11
18
* Fetch all existing play records from Teal
19
+
* Returns a Map where each key can have multiple records (for duplicate detection)
12
20
*/
13
21
export async function fetchExistingRecords(
14
22
agent: AtpAgent,
···
40
48
const playRecord = record.value as PlayRecord;
41
49
// Create a unique key based on track, artist, and timestamp
42
50
const key = createRecordKey(playRecord);
51
+
// Note: This will overwrite duplicates, but that's OK for sync mode
52
+
// For duplicate detection, we'll need to fetch all records again
43
53
existingRecords.set(key, {
44
54
uri: record.uri,
45
55
cid: record.cid,
···
66
76
}
67
77
68
78
/**
79
+
* Fetch all existing play records as an array (for duplicate detection)
80
+
* This version keeps ALL records, including duplicates
81
+
*/
82
+
export async function fetchAllRecords(
83
+
agent: AtpAgent,
84
+
config: Config
85
+
): Promise<ExistingRecord[]> {
86
+
const { RECORD_TYPE } = config;
87
+
const did = agent.session?.did;
88
+
89
+
if (!did) {
90
+
throw new Error('No authenticated session found');
91
+
}
92
+
93
+
ui.startSpinner('Fetching existing records from Teal...');
94
+
const allRecords: ExistingRecord[] = [];
95
+
let cursor: string | undefined = undefined;
96
+
let totalFetched = 0;
97
+
98
+
try {
99
+
// Fetch records in batches using listRecords
100
+
do {
101
+
const response = await agent.com.atproto.repo.listRecords({
102
+
repo: did,
103
+
collection: RECORD_TYPE,
104
+
limit: 100,
105
+
cursor: cursor,
106
+
});
107
+
108
+
for (const record of response.data.records) {
109
+
const playRecord = record.value as PlayRecord;
110
+
allRecords.push({
111
+
uri: record.uri,
112
+
cid: record.cid,
113
+
value: playRecord,
114
+
});
115
+
}
116
+
117
+
totalFetched += response.data.records.length;
118
+
cursor = response.data.cursor;
119
+
120
+
// Update spinner with progress
121
+
if (totalFetched % 500 === 0 && totalFetched > 0) {
122
+
ui.updateSpinner(`Fetching records... ${totalFetched.toLocaleString()} found`);
123
+
}
124
+
} while (cursor);
125
+
126
+
ui.succeedSpinner(`Found ${allRecords.length.toLocaleString()} total records`);
127
+
return allRecords;
128
+
} catch (error) {
129
+
ui.failSpinner('Failed to fetch existing records');
130
+
throw error;
131
+
}
132
+
}
133
+
134
+
/**
69
135
* Create a unique key for a play record based on its essential properties
70
136
* This is used to identify duplicates
71
137
*/
···
111
177
console.log('Examples of existing records (skipped):');
112
178
duplicates.slice(0, 5).forEach((record, i) => {
113
179
console.log(` ${i + 1}. ${record.artists[0]?.artistName} - ${record.trackName}`);
114
-
console.log(` Played: ${record.playedTime}`);
180
+
console.log(` Played: ${formatDate(record.playedTime, true)}`);
115
181
});
116
182
console.log('');
117
183
} else if (duplicates.length > 5) {
118
184
console.log('Examples of existing records (skipped):');
119
185
duplicates.slice(0, 5).forEach((record, i) => {
120
186
console.log(` ${i + 1}. ${record.artists[0]?.artistName} - ${record.trackName}`);
121
-
console.log(` Played: ${record.playedTime}`);
187
+
console.log(` Played: ${formatDate(record.playedTime, true)}`);
122
188
});
123
189
console.log(` ... and ${duplicates.length - 5} more duplicates\n`);
124
190
}
···
157
223
console.log(`Last.fm Export:`);
158
224
console.log(` Total records: ${lastfmRecords.length}`);
159
225
if (lastfmRange) {
160
-
console.log(` Date range: ${lastfmRange.earliest.toLocaleDateString()} to ${lastfmRange.latest.toLocaleDateString()}`);
226
+
console.log(` Date range: ${formatDateRange(lastfmRange.earliest, lastfmRange.latest)}`);
161
227
}
162
228
console.log('');
163
229
164
230
console.log(`Teal (Current):`);
165
231
console.log(` Total records: ${existingRecords.size}`);
166
232
if (existingRange) {
167
-
console.log(` Date range: ${existingRange.earliest.toLocaleDateString()} to ${existingRange.latest.toLocaleDateString()}`);
233
+
console.log(` Date range: ${formatDateRange(existingRange.earliest, existingRange.latest)}`);
168
234
}
169
235
console.log('');
170
236
···
174
240
console.log(` Match rate: ${((1 - newRecords.length / lastfmRecords.length) * 100).toFixed(1)}%`);
175
241
console.log('');
176
242
}
243
+
244
+
/**
245
+
* Find duplicate records in the existing records
246
+
* Returns groups of duplicates (where each group has 2+ records with the same key)
247
+
*/
248
+
export function findDuplicates(
249
+
allRecords: ExistingRecord[]
250
+
): DuplicateGroup[] {
251
+
const keyGroups = new Map<string, ExistingRecord[]>();
252
+
253
+
// Group records by their key
254
+
for (const record of allRecords) {
255
+
const key = createRecordKey(record.value);
256
+
if (!keyGroups.has(key)) {
257
+
keyGroups.set(key, []);
258
+
}
259
+
keyGroups.get(key)!.push(record);
260
+
}
261
+
262
+
// Filter to only groups with duplicates (2+ records)
263
+
const duplicates: DuplicateGroup[] = [];
264
+
for (const [key, records] of keyGroups) {
265
+
if (records.length > 1) {
266
+
duplicates.push({ key, records });
267
+
}
268
+
}
269
+
270
+
return duplicates;
271
+
}
272
+
273
+
/**
274
+
* Remove duplicate records from Teal, keeping only the first occurrence
275
+
*/
276
+
export async function removeDuplicates(
277
+
agent: AtpAgent,
278
+
config: Config,
279
+
dryRun: boolean = false
280
+
): Promise<{ totalDuplicates: number; recordsRemoved: number }> {
281
+
ui.header('Checking for Duplicate Records');
282
+
283
+
// Fetch ALL records (including duplicates)
284
+
const allRecords = await fetchAllRecords(agent, config);
285
+
286
+
ui.startSpinner('Analyzing records for duplicates...');
287
+
const duplicateGroups = findDuplicates(allRecords);
288
+
289
+
if (duplicateGroups.length === 0) {
290
+
ui.succeedSpinner('No duplicates found!');
291
+
return { totalDuplicates: 0, recordsRemoved: 0 };
292
+
}
293
+
294
+
ui.stopSpinner();
295
+
296
+
// Count total duplicate records (excluding the one we keep per group)
297
+
const totalDuplicates = duplicateGroups.reduce((sum, group) => sum + (group.records.length - 1), 0);
298
+
299
+
ui.warning(`Found ${duplicateGroups.length.toLocaleString()} duplicate groups (${totalDuplicates.toLocaleString()} records to remove)`);
300
+
console.log('');
301
+
302
+
// Show examples
303
+
const exampleCount = Math.min(5, duplicateGroups.length);
304
+
ui.subheader('Examples of Duplicates:');
305
+
for (let i = 0; i < exampleCount; i++) {
306
+
const group = duplicateGroups[i];
307
+
const firstRecord = group.records[0].value;
308
+
console.log(` ${i + 1}. ${firstRecord.artists[0]?.artistName} - ${firstRecord.trackName}`);
309
+
console.log(` ${formatDate(firstRecord.playedTime, true)} · ${group.records.length - 1} duplicate(s)`);
310
+
}
311
+
312
+
if (duplicateGroups.length > exampleCount) {
313
+
console.log(` ... and ${duplicateGroups.length - exampleCount} more groups`);
314
+
}
315
+
console.log('');
316
+
317
+
if (dryRun) {
318
+
ui.info('DRY RUN: No records were removed.');
319
+
return { totalDuplicates, recordsRemoved: 0 };
320
+
}
321
+
322
+
// Remove duplicates (keep first, delete rest)
323
+
console.log('');
324
+
const progressBar = ui.createProgressBar(totalDuplicates, 'Removing duplicates');
325
+
let recordsRemoved = 0;
326
+
const startTime = Date.now();
327
+
328
+
for (const group of duplicateGroups) {
329
+
// Keep the first record, delete the rest
330
+
const toDelete = group.records.slice(1);
331
+
332
+
for (const record of toDelete) {
333
+
try {
334
+
await agent.com.atproto.repo.deleteRecord({
335
+
repo: agent.session?.did || '',
336
+
collection: record.value.$type,
337
+
rkey: record.uri.split('/').pop()!,
338
+
});
339
+
recordsRemoved++;
340
+
341
+
// Update progress bar
342
+
const elapsed = (Date.now() - startTime) / 1000;
343
+
const speed = recordsRemoved / Math.max(elapsed, 0.1);
344
+
progressBar.update(recordsRemoved, { speed });
345
+
346
+
} catch (error) {
347
+
// Silently continue on errors
348
+
}
349
+
350
+
// Small delay between deletions
351
+
await new Promise(resolve => setTimeout(resolve, 100));
352
+
}
353
+
}
354
+
355
+
progressBar.stop();
356
+
console.log('');
357
+
ui.success(`Removed ${recordsRemoved.toLocaleString()} duplicate records`);
358
+
ui.info(`Kept ${duplicateGroups.length.toLocaleString()} unique records`);
359
+
360
+
return { totalDuplicates, recordsRemoved };
361
+
}