A decentralized music tracking and discovery platform built on AT Protocol 🎵
0
fork

Configure Feed

Select the types of activity you want to include in your feed.

refactor: restructure deduplication logic to use a do-while loop for batch processing

+32 -24
+32 -24
apps/api/src/scripts/dedup.ts
··· 19 19 } 20 20 21 21 const agent = await createAgent(ctx.oauthClient, did); 22 - const records = await agent.com.atproto.repo.listRecords({ 23 - repo: agent.assertDid, 24 - collection: "app.rocksky.scrobble", 25 - limit: 100, 26 - }); 22 + let cursor: string | undefined; 23 + const BATCH_SIZE = 100; 24 + let i = 1; 25 + do { 26 + const records = await agent.com.atproto.repo.listRecords({ 27 + repo: agent.assertDid, 28 + collection: "app.rocksky.scrobble", 29 + limit: BATCH_SIZE, 30 + }); 27 31 28 - for (const record of records.data.records) { 29 - const result = await ctx.db 30 - .select() 31 - .from(tables.scrobbles) 32 - .where(eq(tables.scrobbles.uri, record.uri)) 33 - .limit(1); 34 - if (result.length === 0) { 35 - console.log("Deleting record:"); 36 - console.log(record); 37 - const rkey = record.uri.split("/").pop(); 38 - await agent.com.atproto.repo.deleteRecord({ 39 - repo: agent.assertDid, 40 - collection: "app.rocksky.scrobble", 41 - rkey, 42 - }); 43 - } else { 44 - console.log(chalk.greenBright("Keeping record:")); 45 - console.log(record); 32 + for (const record of records.data.records) { 33 + const result = await ctx.db 34 + .select() 35 + .from(tables.scrobbles) 36 + .where(eq(tables.scrobbles.uri, record.uri)) 37 + .limit(1); 38 + if (result.length === 0) { 39 + console.log(`${i} Deleting record:`); 40 + console.log(record); 41 + const rkey = record.uri.split("/").pop(); 42 + await agent.com.atproto.repo.deleteRecord({ 43 + repo: agent.assertDid, 44 + collection: "app.rocksky.scrobble", 45 + rkey, 46 + }); 47 + await new Promise((resolve) => setTimeout(resolve, 1000)); // rate limit 48 + } else { 49 + console.log(chalk.greenBright(`${i} Keeping record:`)); 50 + console.log(record); 51 + } 52 + i += 1; 46 53 } 47 - } 54 + cursor = records.data.cursor; 55 + } while (cursor); 48 56 49 57 console.log(chalk.greenBright("Deduplication complete.")); 50 58