forked from
rocksky.app/rocksky
A decentralized music tracking and discovery platform built on AT Protocol 🎵
refactor: restructure deduplication logic to use a do-while loop for batch processing
+32
-24
+32
-24
apps/api/src/scripts/dedup.ts
+32
-24
apps/api/src/scripts/dedup.ts
···
19
19
}
20
20
21
21
const agent = await createAgent(ctx.oauthClient, did);
22
-
const records = await agent.com.atproto.repo.listRecords({
23
-
repo: agent.assertDid,
24
-
collection: "app.rocksky.scrobble",
25
-
limit: 100,
26
-
});
22
+
let cursor: string | undefined;
23
+
const BATCH_SIZE = 100;
24
+
let i = 1;
25
+
do {
26
+
const records = await agent.com.atproto.repo.listRecords({
27
+
repo: agent.assertDid,
28
+
collection: "app.rocksky.scrobble",
29
+
limit: BATCH_SIZE,
30
+
});
27
31
28
-
for (const record of records.data.records) {
29
-
const result = await ctx.db
30
-
.select()
31
-
.from(tables.scrobbles)
32
-
.where(eq(tables.scrobbles.uri, record.uri))
33
-
.limit(1);
34
-
if (result.length === 0) {
35
-
console.log("Deleting record:");
36
-
console.log(record);
37
-
const rkey = record.uri.split("/").pop();
38
-
await agent.com.atproto.repo.deleteRecord({
39
-
repo: agent.assertDid,
40
-
collection: "app.rocksky.scrobble",
41
-
rkey,
42
-
});
43
-
} else {
44
-
console.log(chalk.greenBright("Keeping record:"));
45
-
console.log(record);
32
+
for (const record of records.data.records) {
33
+
const result = await ctx.db
34
+
.select()
35
+
.from(tables.scrobbles)
36
+
.where(eq(tables.scrobbles.uri, record.uri))
37
+
.limit(1);
38
+
if (result.length === 0) {
39
+
console.log(`${i} Deleting record:`);
40
+
console.log(record);
41
+
const rkey = record.uri.split("/").pop();
42
+
await agent.com.atproto.repo.deleteRecord({
43
+
repo: agent.assertDid,
44
+
collection: "app.rocksky.scrobble",
45
+
rkey,
46
+
});
47
+
await new Promise((resolve) => setTimeout(resolve, 1000)); // rate limit
48
+
} else {
49
+
console.log(chalk.greenBright(`${i} Keeping record:`));
50
+
console.log(record);
51
+
}
52
+
i += 1;
46
53
}
47
-
}
54
+
cursor = records.data.cursor;
55
+
} while (cursor);
48
56
49
57
console.log(chalk.greenBright("Deduplication complete."));
50
58