Monorepo for Aesthetic.Computer
aesthetic.computer
feat(at): add backfill scripts for pieces and kidlisp
- backfill-pieces-to-atproto.mjs — syncs all unsynced pieces (186 done)
- backfill-kidlisp-to-atproto.mjs — syncs user + guest kidlisp records
supports --users-only, --guests-only, --dry-run
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
+343
+188
at/scripts/atproto/backfill-kidlisp-to-atproto.mjs
+188
at/scripts/atproto/backfill-kidlisp-to-atproto.mjs
···
1
+
#!/usr/bin/env node
2
+
3
+
/**
4
+
* Backfill KidLisp to ATProto
5
+
*
6
+
* Syncs all existing MongoDB kidlisp records that don't have ATProto records.
7
+
* Handles both user content and guest/anonymous content (via art account).
8
+
*
9
+
* Usage:
10
+
* node backfill-kidlisp-to-atproto.mjs [options]
11
+
*
12
+
* Options:
13
+
* --dry-run Show what would be synced without making changes
14
+
* --limit N Only process N records (for testing)
15
+
* --user @handle Only process for specific user
16
+
* --guests-only Only backfill guest/anonymous records
17
+
* --users-only Only backfill user records (skip guests)
18
+
* --batch-size N Process N records at a time (default: 20)
19
+
* --delay MS Delay between batches in ms (default: 500)
20
+
*/
21
+
22
+
import { connect } from "../../../system/backend/database.mjs";
23
+
import { createMediaRecord, MediaTypes } from "../../../system/backend/media-atproto.mjs";
24
+
import { config } from "dotenv";
25
+
26
+
config({ path: "../../../system/.env" });
27
+
28
+
const args = process.argv.slice(2);
29
+
const dryRun = args.includes("--dry-run");
30
+
const limitIndex = args.indexOf("--limit");
31
+
const limit = limitIndex !== -1 ? parseInt(args[limitIndex + 1]) : null;
32
+
const userIndex = args.indexOf("--user");
33
+
const targetUser =
34
+
userIndex !== -1 ? args[userIndex + 1]?.replace("@", "") : null;
35
+
const guestsOnly = args.includes("--guests-only");
36
+
const usersOnly = args.includes("--users-only");
37
+
const batchSizeIndex = args.indexOf("--batch-size");
38
+
const batchSize =
39
+
batchSizeIndex !== -1 ? parseInt(args[batchSizeIndex + 1]) : 20;
40
+
const delayIndex = args.indexOf("--delay");
41
+
const delay = delayIndex !== -1 ? parseInt(args[delayIndex + 1]) : 500;
42
+
43
+
async function sleep(ms) {
44
+
return new Promise((resolve) => setTimeout(resolve, ms));
45
+
}
46
+
47
+
async function backfillKidlisp() {
48
+
console.log("🎨 Backfill KidLisp to ATProto\n");
49
+
console.log(`Mode: ${dryRun ? "🔍 DRY RUN" : "✍️ LIVE"}`);
50
+
if (limit) console.log(`Limit: ${limit} records`);
51
+
if (targetUser) console.log(`User: @${targetUser}`);
52
+
if (guestsOnly) console.log(`Scope: guests only`);
53
+
if (usersOnly) console.log(`Scope: users only`);
54
+
console.log(`Batch size: ${batchSize}`);
55
+
console.log(`Delay: ${delay}ms`);
56
+
console.log();
57
+
58
+
const database = await connect();
59
+
const kidlisp = database.db.collection("kidlisp");
60
+
const users = database.db.collection("users");
61
+
const handles = database.db.collection("@handles");
62
+
63
+
const query = {
64
+
$or: [
65
+
{ atproto: { $exists: false } },
66
+
{ "atproto.rkey": { $exists: false } },
67
+
],
68
+
};
69
+
70
+
if (guestsOnly) {
71
+
query.$and = [{ $or: [{ user: { $exists: false } }, { user: null }] }];
72
+
} else if (usersOnly) {
73
+
query.user = { $exists: true, $ne: null };
74
+
}
75
+
76
+
if (targetUser) {
77
+
const handleDoc = await handles.findOne({ handle: targetUser });
78
+
if (!handleDoc) {
79
+
console.error(`❌ User @${targetUser} not found`);
80
+
await database.disconnect();
81
+
process.exit(1);
82
+
}
83
+
query.user = handleDoc._id;
84
+
}
85
+
86
+
const cursor = kidlisp.find(query).sort({ when: -1 });
87
+
if (limit) cursor.limit(limit);
88
+
89
+
const allRecords = await cursor.toArray();
90
+
console.log(`Found ${allRecords.length} unsynced kidlisp records\n`);
91
+
92
+
if (allRecords.length === 0) {
93
+
console.log("✨ Nothing to backfill!");
94
+
await database.disconnect();
95
+
return;
96
+
}
97
+
98
+
let synced = 0,
99
+
skipped = 0,
100
+
failed = 0;
101
+
102
+
// Cache user lookups
103
+
const userCache = {};
104
+
105
+
for (let i = 0; i < allRecords.length; i++) {
106
+
const item = allRecords[i];
107
+
const isGuest = !item.user;
108
+
let h = "guest";
109
+
110
+
if (!isGuest) {
111
+
if (!(item.user in userCache)) {
112
+
const handleDoc = await handles.findOne({ _id: item.user });
113
+
userCache[item.user] = handleDoc?.handle || "unknown";
114
+
}
115
+
h = userCache[item.user];
116
+
}
117
+
118
+
const label = item.code
119
+
? item.code.slice(0, 30)
120
+
: item._id.toString().slice(-8);
121
+
122
+
if (dryRun) {
123
+
console.log(
124
+
` [${i + 1}/${allRecords.length}] Would sync: @${h} — ${label}`,
125
+
);
126
+
synced++;
127
+
continue;
128
+
}
129
+
130
+
try {
131
+
const result = await createMediaRecord(
132
+
database,
133
+
MediaTypes.KIDLISP,
134
+
item,
135
+
{ userSub: item.user || null },
136
+
);
137
+
138
+
if (result.error) {
139
+
console.log(
140
+
` [${i + 1}/${allRecords.length}] ⏭️ @${h}: ${result.error}`,
141
+
);
142
+
skipped++;
143
+
} else {
144
+
await kidlisp.updateOne(
145
+
{ _id: item._id },
146
+
{
147
+
$set: {
148
+
"atproto.rkey": result.rkey,
149
+
"atproto.uri": result.uri,
150
+
"atproto.syncedAt": new Date().toISOString(),
151
+
},
152
+
},
153
+
);
154
+
if ((i + 1) % 100 === 0 || i < 5) {
155
+
console.log(
156
+
` [${i + 1}/${allRecords.length}] ✅ @${h} → ${result.rkey}`,
157
+
);
158
+
}
159
+
synced++;
160
+
}
161
+
} catch (error) {
162
+
console.log(
163
+
` [${i + 1}/${allRecords.length}] ❌ @${h}: ${error.message}`,
164
+
);
165
+
failed++;
166
+
}
167
+
168
+
if ((i + 1) % batchSize === 0 && i < allRecords.length - 1) {
169
+
if ((i + 1) % 100 === 0) {
170
+
console.log(` ... ${i + 1}/${allRecords.length} processed (${synced} synced, ${failed} failed)`);
171
+
}
172
+
await sleep(delay);
173
+
}
174
+
}
175
+
176
+
console.log("\n" + "═".repeat(50));
177
+
console.log(`✅ Synced: ${synced}`);
178
+
console.log(`⏭️ Skipped: ${skipped}`);
179
+
console.log(`❌ Failed: ${failed}`);
180
+
console.log(`📊 Total: ${allRecords.length}\n`);
181
+
182
+
await database.disconnect();
183
+
}
184
+
185
+
backfillKidlisp().catch((err) => {
186
+
console.error("Fatal:", err);
187
+
process.exit(1);
188
+
});
+155
at/scripts/atproto/backfill-pieces-to-atproto.mjs
+155
at/scripts/atproto/backfill-pieces-to-atproto.mjs
···
1
+
#!/usr/bin/env node
2
+
3
+
/**
4
+
* Backfill Pieces to ATProto
5
+
*
6
+
* Syncs all existing MongoDB pieces that don't have ATProto records yet.
7
+
* Only syncs for users who have ATProto accounts.
8
+
*
9
+
* Usage:
10
+
* node backfill-pieces-to-atproto.mjs [options]
11
+
*
12
+
* Options:
13
+
* --dry-run Show what would be synced without making changes
14
+
* --limit N Only process N pieces (for testing)
15
+
* --user @handle Only process pieces for specific user
16
+
* --batch-size N Process N pieces at a time (default: 10)
17
+
* --delay MS Delay between batches in ms (default: 1000)
18
+
*/
19
+
20
+
import { connect } from "../../../system/backend/database.mjs";
21
+
import { createMediaRecord, MediaTypes } from "../../../system/backend/media-atproto.mjs";
22
+
import { config } from "dotenv";
23
+
24
+
config({ path: "../../../system/.env" });
25
+
26
+
const args = process.argv.slice(2);
27
+
const dryRun = args.includes("--dry-run");
28
+
const limitIndex = args.indexOf("--limit");
29
+
const limit = limitIndex !== -1 ? parseInt(args[limitIndex + 1]) : null;
30
+
const userIndex = args.indexOf("--user");
31
+
const targetUser =
32
+
userIndex !== -1 ? args[userIndex + 1]?.replace("@", "") : null;
33
+
const batchSizeIndex = args.indexOf("--batch-size");
34
+
const batchSize =
35
+
batchSizeIndex !== -1 ? parseInt(args[batchSizeIndex + 1]) : 10;
36
+
const delayIndex = args.indexOf("--delay");
37
+
const delay = delayIndex !== -1 ? parseInt(args[delayIndex + 1]) : 1000;
38
+
39
+
async function sleep(ms) {
40
+
return new Promise((resolve) => setTimeout(resolve, ms));
41
+
}
42
+
43
+
async function backfillPieces() {
44
+
console.log("🧩 Backfill Pieces to ATProto\n");
45
+
console.log(`Mode: ${dryRun ? "🔍 DRY RUN" : "✍️ LIVE"}`);
46
+
if (limit) console.log(`Limit: ${limit} pieces`);
47
+
if (targetUser) console.log(`User: @${targetUser}`);
48
+
console.log(`Batch size: ${batchSize}`);
49
+
console.log(`Delay: ${delay}ms`);
50
+
console.log();
51
+
52
+
const database = await connect();
53
+
const pieces = database.db.collection("pieces");
54
+
const users = database.db.collection("users");
55
+
const handles = database.db.collection("@handles");
56
+
57
+
// Build query for pieces without ATProto records
58
+
const query = {
59
+
user: { $exists: true, $ne: null },
60
+
$or: [{ atproto: { $exists: false } }, { "atproto.rkey": { $exists: false } }],
61
+
};
62
+
63
+
// Filter by user handle if specified
64
+
if (targetUser) {
65
+
const handleDoc = await handles.findOne({ handle: targetUser });
66
+
if (!handleDoc) {
67
+
console.error(`❌ User @${targetUser} not found`);
68
+
await database.disconnect();
69
+
process.exit(1);
70
+
}
71
+
query.user = handleDoc._id;
72
+
}
73
+
74
+
const cursor = pieces.find(query).sort({ when: -1 });
75
+
if (limit) cursor.limit(limit);
76
+
77
+
const allPieces = await cursor.toArray();
78
+
console.log(`Found ${allPieces.length} unsynced pieces\n`);
79
+
80
+
if (allPieces.length === 0) {
81
+
console.log("✨ Nothing to backfill!");
82
+
await database.disconnect();
83
+
return;
84
+
}
85
+
86
+
let synced = 0,
87
+
skipped = 0,
88
+
failed = 0;
89
+
90
+
for (let i = 0; i < allPieces.length; i++) {
91
+
const piece = allPieces[i];
92
+
const handle = await handles.findOne({ _id: piece.user });
93
+
const h = handle?.handle || "unknown";
94
+
95
+
if (dryRun) {
96
+
console.log(
97
+
` [${i + 1}/${allPieces.length}] Would sync: @${h} — ${piece.slug || piece.name || piece._id}`,
98
+
);
99
+
synced++;
100
+
continue;
101
+
}
102
+
103
+
try {
104
+
const result = await createMediaRecord(database, MediaTypes.PIECE, piece, {
105
+
userSub: piece.user,
106
+
});
107
+
108
+
if (result.error) {
109
+
console.log(
110
+
` [${i + 1}/${allPieces.length}] ⏭️ @${h} — ${piece.slug}: ${result.error}`,
111
+
);
112
+
skipped++;
113
+
} else {
114
+
// Store rkey back in MongoDB
115
+
await pieces.updateOne(
116
+
{ _id: piece._id },
117
+
{
118
+
$set: {
119
+
"atproto.rkey": result.rkey,
120
+
"atproto.uri": result.uri,
121
+
"atproto.syncedAt": new Date().toISOString(),
122
+
},
123
+
},
124
+
);
125
+
console.log(
126
+
` [${i + 1}/${allPieces.length}] ✅ @${h} — ${piece.slug} → ${result.rkey}`,
127
+
);
128
+
synced++;
129
+
}
130
+
} catch (error) {
131
+
console.log(
132
+
` [${i + 1}/${allPieces.length}] ❌ @${h} — ${piece.slug}: ${error.message}`,
133
+
);
134
+
failed++;
135
+
}
136
+
137
+
// Batch delay
138
+
if ((i + 1) % batchSize === 0 && i < allPieces.length - 1) {
139
+
await sleep(delay);
140
+
}
141
+
}
142
+
143
+
console.log("\n" + "═".repeat(50));
144
+
console.log(`✅ Synced: ${synced}`);
145
+
console.log(`⏭️ Skipped: ${skipped}`);
146
+
console.log(`❌ Failed: ${failed}`);
147
+
console.log(`📊 Total: ${allPieces.length}\n`);
148
+
149
+
await database.disconnect();
150
+
}
151
+
152
+
backfillPieces().catch((err) => {
153
+
console.error("Fatal:", err);
154
+
process.exit(1);
155
+
});