Monorepo for Aesthetic.Computer
aesthetic.computer
1#!/usr/bin/env node
2// sync-atproto.mjs
3// Master ATProto synchronization script
4// Handles all content types: paintings, moods, pieces, kidlisp
5//
6// Features:
7// 1. Cleans up duplicate ATProto records (keeps earliest by ref)
8// 2. Syncs MongoDB → ATProto for missing records
9// 3. Never creates duplicates
10// 4. Can be run multiple times safely
11// 5. Handles anonymous content → art account
12// 6. Handles user content → personal accounts
13
14import { connect } from "../../system/backend/database.mjs";
15import { AtpAgent } from "@atproto/api";
16import {
17 MediaTypes,
18 createMediaRecord,
19 deleteMediaRecord,
20 getCollection,
21 supportsAnonymous
22} from "../../system/backend/media-atproto.mjs";
23
24const PDS_URL = process.env.PDS_URL || "https://at.aesthetic.computer";
25
26// Collection configurations - now using unified media-atproto module
27const COLLECTIONS = {
28 paintings: {
29 mediaType: MediaTypes.PAINTING,
30 mongoCollection: "paintings",
31 atprotoCollection: getCollection(MediaTypes.PAINTING),
32 hasAnonymous: supportsAnonymous(MediaTypes.PAINTING)
33 },
34 moods: {
35 mediaType: MediaTypes.MOOD,
36 mongoCollection: "moods",
37 atprotoCollection: getCollection(MediaTypes.MOOD),
38 hasAnonymous: supportsAnonymous(MediaTypes.MOOD)
39 },
40 pieces: {
41 mediaType: MediaTypes.PIECE,
42 mongoCollection: "pieces",
43 atprotoCollection: getCollection(MediaTypes.PIECE),
44 hasAnonymous: supportsAnonymous(MediaTypes.PIECE)
45 },
46 kidlisp: {
47 mediaType: MediaTypes.KIDLISP,
48 mongoCollection: "kidlisp",
49 atprotoCollection: getCollection(MediaTypes.KIDLISP),
50 hasAnonymous: supportsAnonymous(MediaTypes.KIDLISP)
51 },
52 tapes: {
53 mediaType: MediaTypes.TAPE,
54 mongoCollection: "tapes",
55 atprotoCollection: getCollection(MediaTypes.TAPE),
56 hasAnonymous: supportsAnonymous(MediaTypes.TAPE)
57 }
58};
59
60/**
61 * Wipe all ATProto data for a specific user (including anonymous/art-guest)
62 * - Deletes all ATProto records
63 * - Clears atproto.rkey fields from MongoDB
64 * - Supports art.at.aesthetic.computer for anonymous content
65 */
66async function wipeUserAtprotoData(userHandle, options = {}) {
67 const { dryRun = true, contentTypes = ['paintings', 'moods', 'pieces', 'kidlisp', 'tapes'] } = options;
68
69 console.log(`\n🗑️ WIPE MODE: ${userHandle}`);
70 console.log(` Mode: ${dryRun ? '🔍 DRY RUN (no changes)' : '⚡ LIVE (deleting)'}`);
71 console.log(` Content types: ${contentTypes.join(', ')}\n`);
72
73 const database = await connect();
74 const users = database.db.collection("users");
75
76 // Find user by handle
77 const user = await users.findOne({ "atproto.handle": userHandle });
78
79 if (!user) {
80 console.error(`❌ User not found: ${userHandle}`);
81 await database.disconnect();
82 return;
83 }
84
85 if (!user.atproto?.did || !user.atproto?.password) {
86 console.error(`❌ User has no ATProto account: ${userHandle}`);
87 await database.disconnect();
88 return;
89 }
90
91 console.log(`📋 Found user: ${user.atproto.handle} (${user.atproto.did})`);
92
93 // Login to ATProto
94 const agent = new AtpAgent({ service: PDS_URL });
95 await agent.login({
96 identifier: user.atproto.did,
97 password: user.atproto.password
98 });
99
100 const stats = {
101 deleted: 0,
102 mongoCleared: 0,
103 errors: []
104 };
105
106 // Process each requested content type
107 for (const [contentType, config] of Object.entries(COLLECTIONS)) {
108 if (!contentTypes.includes(contentType)) {
109 continue; // Skip this type
110 }
111
112 console.log(`\n Processing ${contentType}...`);
113
114 // 1. Fetch all ATProto records for this collection
115 let atprotoRecords = [];
116 let cursor = undefined;
117
118 while (true) {
119 const response = await agent.com.atproto.repo.listRecords({
120 repo: user.atproto.did,
121 collection: config.atprotoCollection,
122 limit: 100,
123 cursor
124 });
125
126 atprotoRecords.push(...response.data.records);
127 if (!response.data.cursor) break;
128 cursor = response.data.cursor;
129 }
130
131 console.log(` Found ${atprotoRecords.length} ATProto records`);
132
133 // 2. Delete all ATProto records
134 if (atprotoRecords.length > 0) {
135 for (const record of atprotoRecords) {
136 const rkey = record.uri.split("/").pop();
137
138 if (!dryRun) {
139 try {
140 await agent.com.atproto.repo.deleteRecord({
141 repo: user.atproto.did,
142 collection: config.atprotoCollection,
143 rkey
144 });
145 stats.deleted++;
146 } catch (error) {
147 console.log(` ❌ Failed to delete ${rkey}: ${error.message}`);
148 stats.errors.push(`Failed to delete ${contentType}/${rkey}: ${error.message}`);
149 }
150 } else {
151 stats.deleted++;
152 }
153 }
154 console.log(` ${dryRun ? 'Would delete' : 'Deleted'} ${atprotoRecords.length} records from ATProto`);
155 }
156
157 // 3. Clear atproto.rkey fields from MongoDB
158 const collection = database.db.collection(config.mongoCollection);
159 // For anonymous content (art-guest), match records WITHOUT user field
160 // For user content, match specific user._id
161 const isAnonymous = user._id === "art-guest";
162 const mongoQuery = isAnonymous
163 ? { user: { $exists: false }, "atproto.rkey": { $exists: true } }
164 : { user: user._id, "atproto.rkey": { $exists: true } };
165 const mongoCount = await collection.countDocuments(mongoQuery);
166
167 if (mongoCount > 0) {
168 if (!dryRun) {
169 const result = await collection.updateMany(
170 mongoQuery,
171 { $unset: { "atproto.rkey": "" } }
172 );
173 stats.mongoCleared += result.modifiedCount;
174 } else {
175 stats.mongoCleared += mongoCount;
176 }
177 console.log(` ${dryRun ? 'Would clear' : 'Cleared'} atproto.rkey from ${mongoCount} MongoDB records`);
178 }
179 }
180
181 // Summary
182 console.log(`\n✨ WIPE SUMMARY:`);
183 console.log(` ATProto records deleted: ${stats.deleted}`);
184 console.log(` MongoDB rkeys cleared: ${stats.mongoCleared}`);
185
186 if (stats.errors.length > 0) {
187 console.log(` Errors: ${stats.errors.length}`);
188 stats.errors.forEach(err => console.log(` ${err}`));
189 }
190
191 if (dryRun) {
192 console.log(`\n💡 Run with 'live' argument to apply changes`);
193 }
194
195 await database.disconnect();
196}
197
198/**
199 * Restore ATProto data for a specific user from MongoDB (including anonymous/art-guest)
200 * - Creates ATProto records for all MongoDB items that don't have them
201 * - Updates MongoDB with rkeys
202 * - Supports art.at.aesthetic.computer for anonymous content
203 */
204async function restoreUserAtprotoData(userHandle, options = {}) {
205 const { dryRun = true, contentTypes = ['paintings', 'moods', 'pieces', 'kidlisp', 'tapes'] } = options;
206
207 console.log(`\n♻️ RESTORE MODE: ${userHandle}`);
208 console.log(` Mode: ${dryRun ? '🔍 DRY RUN (no changes)' : '⚡ LIVE (creating)'}`);
209 console.log(` Content types: ${contentTypes.join(', ')}\n`);
210
211 const database = await connect();
212 const users = database.db.collection("users");
213
214 // Find user by handle
215 const user = await users.findOne({ "atproto.handle": userHandle });
216
217 if (!user) {
218 console.error(`❌ User not found: ${userHandle}`);
219 await database.disconnect();
220 return;
221 }
222
223 if (!user.atproto?.did || !user.atproto?.password) {
224 console.error(`❌ User has no ATProto account: ${userHandle}`);
225 await database.disconnect();
226 return;
227 }
228
229 console.log(`📋 Found user: ${user.atproto.handle} (${user.atproto.did})`);
230
231 const allStats = [];
232
233 // Process each requested content type
234 for (const contentType of contentTypes) {
235 if (!COLLECTIONS[contentType]) {
236 console.log(` ⚠️ Unknown content type: ${contentType}, skipping`);
237 continue;
238 }
239
240 console.log(`\n Syncing ${contentType}...`);
241 const stats = await syncContentForUser(contentType, user, { dryRun, verbose: false });
242 allStats.push(stats);
243
244 console.log(` MongoDB: ${stats.mongoTotal}, ATProto: ${stats.atprotoTotal}`);
245 console.log(` Duplicates cleaned: ${stats.duplicatesDeleted}`);
246 console.log(` Missing blobs fixed: ${stats.blobsFixed}`);
247 console.log(` Records ${dryRun ? 'to create' : 'created'}: ${stats.created}`);
248
249 if (stats.errors.length > 0) {
250 console.log(` ❌ Errors: ${stats.errors.length}`);
251 stats.errors.slice(0, 3).forEach(err => console.log(` ${err}`));
252 }
253 }
254
255 // Summary
256 const totalCreated = allStats.reduce((sum, s) => sum + s.created, 0);
257 const totalDuplicates = allStats.reduce((sum, s) => sum + s.duplicatesDeleted, 0);
258 const totalBlobsFixed = allStats.reduce((sum, s) => sum + s.blobsFixed, 0);
259 const totalErrors = allStats.reduce((sum, s) => sum + s.errors.length, 0);
260
261 console.log(`\n✨ RESTORE SUMMARY:`);
262 console.log(` Records ${dryRun ? 'to create' : 'created'}: ${totalCreated}`);
263 console.log(` Duplicates cleaned: ${totalDuplicates}`);
264 console.log(` Missing blobs fixed: ${totalBlobsFixed}`);
265
266 if (totalErrors > 0) {
267 console.log(` Errors: ${totalErrors}`);
268 }
269
270 if (dryRun) {
271 console.log(`\n💡 Run with 'live' argument to apply changes`);
272 }
273
274 await database.disconnect();
275}
276
277/**
278 * Sync a specific content type for a user
279 */
280async function syncContentForUser(contentType, user, options = {}) {
281 const { dryRun = true, verbose = false } = options;
282 const config = COLLECTIONS[contentType];
283 const isAnonymous = user._id === "art-guest";
284
285 const stats = {
286 contentType,
287 handle: user.atproto.handle,
288 mongoTotal: 0,
289 atprotoTotal: 0,
290 duplicatesFound: 0,
291 duplicatesDeleted: 0,
292 missingBlobs: 0,
293 blobsFixed: 0,
294 missingInAtproto: 0,
295 created: 0,
296 errors: []
297 };
298
299 try {
300 const database = await connect();
301 const collection = database.db.collection(config.mongoCollection);
302
303 // Login to ATProto
304 const agent = new AtpAgent({ service: PDS_URL });
305
306 await agent.login({
307 identifier: user.atproto.did,
308 password: user.atproto.password
309 });
310
311 // 1. Fetch all MongoDB items for this user
312 // Anonymous content: no user field. User content: match user._id
313 const mongoQuery = isAnonymous
314 ? { user: { $exists: false } }
315 : { user: user._id };
316
317 const mongoItems = await collection.find(mongoQuery).sort({ when: 1 }).toArray();
318 stats.mongoTotal = mongoItems.length;
319
320 // 2. Fetch all ATProto records
321 let atprotoRecords = [];
322 let cursor = undefined;
323
324 while (true) {
325 const response = await agent.com.atproto.repo.listRecords({
326 repo: user.atproto.did,
327 collection: config.atprotoCollection,
328 limit: 100,
329 cursor
330 });
331
332 atprotoRecords.push(...response.data.records);
333 if (!response.data.cursor) break;
334 cursor = response.data.cursor;
335 }
336
337 stats.atprotoTotal = atprotoRecords.length;
338
339 // 3. Build a map of ATProto records by ref (MongoDB _id)
340 const atprotoByRef = new Map();
341 const atprotoByRkey = new Map();
342 const duplicateRefs = new Set();
343 const recordsWithoutBlobs = [];
344
345 for (const record of atprotoRecords) {
346 const ref = record.value.ref;
347 const rkey = record.uri.split("/").pop();
348
349 atprotoByRkey.set(rkey, record);
350
351 // Check if record has required blobs (thumbnail for paintings/moods, video for tapes)
352 // Note: thumbnail is optional in lexicon, but we want to ensure it exists if the painting exists
353 // The @atproto/api library deserializes blobs, so thumbnail.ref will be a CID object, not {$link: "..."}
354 if (config.mediaType === MediaTypes.PAINTING || config.mediaType === MediaTypes.MOOD) {
355 const hasThumbnail = record.value.thumbnail &&
356 record.value.thumbnail.ref &&
357 typeof record.value.thumbnail.ref !== 'undefined';
358
359 // Only flag as missing blob if the MongoDB record exists
360 // We'll validate this after fetching MongoDB records
361 if (!hasThumbnail && ref) {
362 recordsWithoutBlobs.push({ record, rkey, ref });
363 }
364 }
365
366 // Check for missing video blob on tapes
367 if (config.mediaType === MediaTypes.TAPE) {
368 const hasVideo = record.value.video &&
369 record.value.video.ref &&
370 typeof record.value.video.ref !== 'undefined';
371
372 if (!hasVideo && ref) {
373 recordsWithoutBlobs.push({ record, rkey, ref });
374 }
375 }
376
377 if (ref) {
378 if (atprotoByRef.has(ref)) {
379 // Found a duplicate!
380 duplicateRefs.add(ref);
381 stats.duplicatesFound++;
382 } else {
383 atprotoByRef.set(ref, record);
384 }
385 }
386 }
387
388 // 4. Delete duplicates (keep only the first one for each ref)
389 if (duplicateRefs.size > 0) {
390 for (const ref of duplicateRefs) {
391 // Find all records with this ref
392 const dupes = atprotoRecords.filter(r => r.value.ref === ref);
393
394 // Sort by creation time (earliest first) and keep the first one
395 dupes.sort((a, b) => new Date(a.value.when) - new Date(b.value.when));
396 const toDelete = dupes.slice(1); // Delete all but the first
397
398 // Delete in batches of 50
399 const DELETE_BATCH_SIZE = 50;
400 for (let i = 0; i < toDelete.length; i += DELETE_BATCH_SIZE) {
401 const batch = toDelete.slice(i, i + DELETE_BATCH_SIZE);
402
403 if (!dryRun) {
404 await Promise.all(batch.map(async (dupe) => {
405 const rkey = dupe.uri.split("/").pop();
406
407 try {
408 await agent.com.atproto.repo.deleteRecord({
409 repo: user.atproto.did,
410 collection: config.atprotoCollection,
411 rkey
412 });
413 stats.duplicatesDeleted++;
414 } catch (error) {
415 stats.errors.push(`${user.atproto?.handle || 'anonymous'}: Failed to delete duplicate ${rkey}: ${error.message}`);
416 }
417 }));
418 } else {
419 stats.duplicatesDeleted += batch.length;
420 }
421 }
422 }
423 }
424
425 // 5. Fix records with missing blobs (delete and mark for recreation)
426 // First, validate which records actually exist in MongoDB
427 const { ObjectId } = await import('mongodb');
428 const validRecordsWithoutBlobs = [];
429
430 for (const item of recordsWithoutBlobs) {
431 try {
432 const painting = await collection.findOne({ _id: new ObjectId(item.ref) });
433 if (painting) {
434 validRecordsWithoutBlobs.push(item);
435 }
436 } catch (error) {
437 // ref might not be a valid ObjectId - skip it
438 }
439 }
440
441 stats.missingBlobs = validRecordsWithoutBlobs.length;
442
443 if (validRecordsWithoutBlobs.length > 0) {
444 if (verbose) {
445 console.log(` Found ${validRecordsWithoutBlobs.length} records with missing blobs`);
446 }
447
448 // Delete the incomplete ATProto records
449 const DELETE_BATCH_SIZE = 50;
450 for (let i = 0; i < validRecordsWithoutBlobs.length; i += DELETE_BATCH_SIZE) {
451 const batch = validRecordsWithoutBlobs.slice(i, i + DELETE_BATCH_SIZE);
452
453 if (!dryRun) {
454 await Promise.all(batch.map(async ({ rkey, ref }) => {
455 try {
456 await agent.com.atproto.repo.deleteRecord({
457 repo: user.atproto.did,
458 collection: config.atprotoCollection,
459 rkey
460 });
461
462 // Remove from atprotoByRef map so it will be recreated
463 if (ref) {
464 atprotoByRef.delete(ref);
465 }
466
467 // Clear the rkey from MongoDB so it will be updated
468 try {
469 await collection.updateOne(
470 { _id: new ObjectId(ref) },
471 { $unset: { "atproto.rkey": "" } }
472 );
473 } catch (mongoError) {
474 // ref might not be a valid ObjectId or record doesn't exist
475 }
476
477 stats.blobsFixed++;
478 } catch (error) {
479 stats.errors.push(`${user.atproto?.handle || 'anonymous'}: Failed to delete incomplete record ${rkey}: ${error.message}`);
480 }
481 }));
482 } else {
483 stats.blobsFixed += batch.length;
484 // In dry run, also remove from map to show what would be recreated
485 batch.forEach(({ ref }) => {
486 if (ref) atprotoByRef.delete(ref);
487 });
488 }
489 }
490 }
491
492 // 6. Find MongoDB items that don't exist in ATProto
493 const missingInAtproto = [];
494
495 for (const item of mongoItems) {
496 const refId = item._id.toString();
497
498 // Check if this item exists in ATProto by ref
499 if (!atprotoByRef.has(refId)) {
500 // Check if MongoDB has an atproto.rkey that actually exists
501 if (item.atproto?.rkey && atprotoByRkey.has(item.atproto.rkey)) {
502 // The rkey exists, record is there but maybe needs ref update
503 continue;
504 }
505
506 missingInAtproto.push(item);
507 stats.missingInAtproto++;
508 }
509 }
510
511 // 7. Create missing items in ATProto (includes items with deleted incomplete records)
512 if (missingInAtproto.length > 0) {
513 const mongoUpdates = [];
514
515 // Process in batches of 10 concurrent creates (reduced to avoid rate limiting)
516 const BATCH_SIZE = 10;
517 for (let i = 0; i < missingInAtproto.length; i += BATCH_SIZE) {
518 const batch = missingInAtproto.slice(i, i + BATCH_SIZE);
519
520 if (!dryRun) {
521 await Promise.all(batch.map(async (item, idx) => {
522 const itemName = item.slug || item.text?.substring(0, 30) || item.code || item._id.toString();
523
524 try {
525 // Use unified media-atproto module
526 const result = await createMediaRecord(
527 database,
528 config.mediaType,
529 item,
530 { userSub: isAnonymous ? null : user._id }
531 );
532
533 if (result.error) {
534 throw new Error(result.error);
535 }
536
537 stats.created++;
538
539 // Queue MongoDB update
540 if (result.rkey) {
541 mongoUpdates.push({
542 updateOne: {
543 filter: { _id: item._id },
544 update: { $set: { "atproto.rkey": result.rkey } }
545 }
546 });
547 }
548
549 } catch (error) {
550 stats.errors.push(`${user.atproto?.handle || 'anonymous'}: Failed to create ${itemName}: ${error.message}`);
551 }
552 }));
553 } else {
554 stats.created += batch.length;
555 }
556 }
557
558 // Batch update MongoDB
559 if (mongoUpdates.length > 0 && !dryRun) {
560 try {
561 await collection.bulkWrite(mongoUpdates);
562 } catch (error) {
563 stats.errors.push(`${user.atproto?.handle || 'anonymous'}: Failed to bulk update MongoDB: ${error.message}`);
564 }
565 }
566 }
567
568 await database.disconnect();
569
570 } catch (error) {
571 stats.errors.push(`Fatal error: ${error.message}`);
572 }
573
574 return stats;
575}
576
577/**
578 * Sync all content types
579 */
580async function syncAll(options = {}) {
581 const {
582 dryRun = true,
583 verbose = false,
584 userFilter = null,
585 contentTypes = ['paintings', 'moods', 'pieces', 'kidlisp']
586 } = options;
587
588 console.log(`\n🔄 ATPROTO MASTER SYNC SCRIPT`);
589 console.log(` Mode: ${dryRun ? '🔍 DRY RUN (no changes)' : '⚡ LIVE (making changes)'}`);
590 console.log(` Content types: ${contentTypes.join(', ')}`);
591 console.log(` Time: ${new Date().toISOString()}\n`);
592
593 const database = await connect();
594 const allStats = [];
595
596 // STEP 1: Sync anonymous content to art account
597 const artUser = await database.db.collection("users").findOne({ _id: "art-guest" });
598
599 if (!artUser) {
600 console.error(`\n⚠️ Art account (art-guest) not found in MongoDB!`);
601 console.error(` Run: cd at && node scripts/setup-art-account.mjs\n`);
602 } else {
603 console.log(`\n🎨 Step 1: Syncing anonymous content to ${artUser.atproto.handle}...`);
604
605 for (const contentType of contentTypes) {
606 const config = COLLECTIONS[contentType];
607
608 if (!config.hasAnonymous) {
609 if (verbose) console.log(` Skipping ${contentType} (no anonymous content)`);
610 continue;
611 }
612
613 // Create a pseudo-user object for anonymous content
614 const anonymousUser = {
615 _id: null, // null user means anonymous
616 atproto: artUser.atproto
617 };
618
619 const stats = await syncContentForUser(contentType, anonymousUser, { dryRun, verbose });
620 allStats.push(stats);
621
622 console.log(` ${contentType.padEnd(10)}: ${stats.mongoTotal} in MongoDB, ${stats.atprotoTotal} in ATProto, ${stats.duplicatesDeleted} dupes, ${stats.created} to create`);
623
624 if (stats.errors.length > 0) {
625 console.log(` ${''.padEnd(10)} ⚠️ ${stats.errors.length} errors`);
626 }
627 }
628 }
629
630 // STEP 2: Sync user content
631 if (anonymousOnly) {
632 console.log(`\n👥 Step 2: Skipping user content (--anonymous-only flag)\n`);
633 } else {
634 console.log(`\n👥 Step 2: Syncing user content...`);
635
636 const query = { "atproto.did": { $exists: true }, _id: { $ne: "art-guest" } };
637 if (userFilter) {
638 query["atproto.handle"] = new RegExp(userFilter, "i");
639 }
640
641 const users = await database.db.collection("users").find(query).toArray();
642 console.log(`📋 Found ${users.length} users with ATProto accounts\n`);
643
644 // Process users one at a time to avoid ATProto PDS rate limiting
645 const USER_BATCH_SIZE = 1;
646 let completedUsers = 0;
647 const startTime = Date.now();
648
649 for (let i = 0; i < users.length; i += USER_BATCH_SIZE) {
650 const userBatch = users.slice(i, i + USER_BATCH_SIZE);
651 const batchStart = Date.now();
652
653 // Track batch stats
654 let batchCreated = 0;
655 let batchDuplicates = 0;
656 let batchErrors = 0;
657
658 await Promise.all(userBatch.map(async (user) => {
659 const userStats = [];
660
661 for (const contentType of contentTypes) {
662 const stats = await syncContentForUser(contentType, user, { dryRun, verbose });
663 allStats.push(stats);
664 userStats.push(stats);
665
666 // Aggregate batch stats
667 batchCreated += stats.created;
668 batchDuplicates += stats.duplicatesDeleted;
669 batchErrors += stats.errors.length;
670 }
671
672 // Show summary for this user inline only if there's activity
673 const userIssues = userStats.filter(s => s.duplicatesDeleted > 0 || s.created > 0 || s.errors.length > 0);
674
675 completedUsers++;
676
677 if (userIssues.length > 0) {
678 const summary = userIssues.map(s => {
679 const parts = [];
680 if (s.duplicatesDeleted > 0) parts.push(`-${s.duplicatesDeleted}`);
681 if (s.created > 0) parts.push(`+${s.created}`);
682 if (s.errors.length > 0) parts.push(`❌${s.errors.length}`);
683 return `${s.contentType}:${parts.join(',')}`;
684 }).join(' ');
685 console.log(` ${user.atproto.handle.padEnd(40)} ${summary}`);
686 }
687 }));
688
689 // Batch summary
690 const batchTime = ((Date.now() - batchStart) / 1000).toFixed(1);
691 const totalTime = ((Date.now() - startTime) / 1000).toFixed(0);
692 const usersPerSec = (completedUsers / (Date.now() - startTime) * 1000).toFixed(1);
693 const eta = users.length > completedUsers
694 ? ((users.length - completedUsers) / usersPerSec).toFixed(0) + 's'
695 : 'done';
696
697 console.log(` 📊 Batch ${Math.floor(i / USER_BATCH_SIZE) + 1}: ${userBatch.length} users in ${batchTime}s | Progress: ${completedUsers}/${users.length} (${usersPerSec}/s) | Created: ${batchCreated} | Dupes: ${batchDuplicates} | Errors: ${batchErrors} | ETA: ${eta}`);
698 }
699
700 console.log('\n');
701 } // End of if (!anonymousOnly) block
702
703 // STEP 3: Print summary
704 console.log(`\n📊 SUMMARY BY CONTENT TYPE`);
705
706 for (const contentType of contentTypes) {
707 const typeStats = allStats.filter(s => s.contentType === contentType);
708 if (typeStats.length === 0) continue;
709
710 const totalMongo = typeStats.reduce((sum, s) => sum + s.mongoTotal, 0);
711 const totalAtproto = typeStats.reduce((sum, s) => sum + s.atprotoTotal, 0);
712 const totalDuplicates = typeStats.reduce((sum, s) => sum + s.duplicatesDeleted, 0);
713 const totalMissingBlobs = typeStats.reduce((sum, s) => sum + s.missingBlobs, 0);
714 const totalBlobsFixed = typeStats.reduce((sum, s) => sum + s.blobsFixed, 0);
715 const totalCreated = typeStats.reduce((sum, s) => sum + s.created, 0);
716 const totalErrors = typeStats.reduce((sum, s) => sum + s.errors.length, 0);
717
718 console.log(`\n${contentType.toUpperCase()}:`);
719 console.log(` MongoDB records: ${totalMongo}`);
720 console.log(` ATProto records: ${totalAtproto}`);
721 console.log(` Duplicates to delete: ${totalDuplicates}`);
722 console.log(` Missing blobs to fix: ${totalMissingBlobs} (will delete & recreate: ${totalBlobsFixed})`);
723 console.log(` Records to create: ${totalCreated}`);
724 if (totalErrors > 0) {
725 console.log(` Errors: ${totalErrors}`);
726 }
727
728 // Show top users with issues
729 const usersWithIssues = typeStats.filter(s => s.duplicatesDeleted > 0 || s.created > 0);
730 if (usersWithIssues.length > 0 && verbose) {
731 console.log(` Users with issues: ${usersWithIssues.length}`);
732 usersWithIssues.slice(0, 5).forEach(s => {
733 console.log(` ${s.handle.padEnd(45)} dupes:${s.duplicatesDeleted} missing:${s.created}`);
734 });
735 }
736 }
737
738 // Overall summary
739 const totalDuplicates = allStats.reduce((sum, s) => sum + s.duplicatesDeleted, 0);
740 const totalCreated = allStats.reduce((sum, s) => sum + s.created, 0);
741 const totalErrors = allStats.reduce((sum, s) => sum + s.errors.length, 0);
742
743 console.log(`\n✨ OVERALL:`);
744 console.log(` Total duplicates to delete: ${totalDuplicates}`);
745 console.log(` Total records to create: ${totalCreated}`);
746 if (totalErrors > 0) {
747 console.log(` Total errors: ${totalErrors}`);
748
749 // Group errors by type
750 const allErrors = allStats.flatMap(s => s.errors);
751 const errorGroups = {};
752 for (const error of allErrors) {
753 // Extract error type from message
754 const match = error.match(/: (.+?):/);
755 const type = match ? match[1] : 'Unknown';
756 errorGroups[type] = (errorGroups[type] || 0) + 1;
757 }
758
759 console.log(`\n Error breakdown:`);
760 Object.entries(errorGroups).forEach(([type, count]) => {
761 console.log(` ${type}: ${count}`);
762 });
763
764 // Show sample errors
765 console.log(`\n Sample errors:`);
766 allErrors.slice(0, 10).forEach(err => {
767 console.log(` ${err}`);
768 });
769 }
770
771 if (dryRun) {
772 console.log(`\n💡 Run with 'live' argument to apply changes`);
773 }
774
775 await database.disconnect();
776}
777
778// Parse command line arguments
779const args = process.argv.slice(2);
780const mode = args.find(a => a === 'live' || a === 'dry-run') || 'dry-run';
781const verbose = args.includes('--verbose') || args.includes('-v');
782const userFilter = args.find(a => a.startsWith('--user='))?.split('=')[1];
783const wipeUser = args.find(a => a.startsWith('--wipe='))?.split('=')[1];
784const restoreUser = args.find(a => a.startsWith('--restore='))?.split('=')[1];
785const anonymousOnly = args.includes('--anonymous-only');
786
787// Parse content types
788let contentTypes = ['paintings', 'moods', 'pieces', 'kidlisp', 'tapes'];
789const contentTypeArg = args.find(a => a.startsWith('--types='));
790if (contentTypeArg) {
791 contentTypes = contentTypeArg.split('=')[1].split(',');
792}
793
794// Single type shortcuts
795if (args.includes('--paintings-only')) contentTypes = ['paintings'];
796if (args.includes('--moods-only')) contentTypes = ['moods'];
797if (args.includes('--pieces-only')) contentTypes = ['pieces'];
798if (args.includes('--kidlisp-only')) contentTypes = ['kidlisp'];
799if (args.includes('--tapes-only')) contentTypes = ['tapes'];
800
801const dryRun = mode !== 'live';
802
803// Show usage if --help
804if (args.includes('--help') || args.includes('-h')) {
805 console.log(`
806🔄 ATProto Master Sync Script
807
808Usage: node sync-atproto.mjs [mode] [options]
809
810Modes:
811 dry-run Show what would be changed (default)
812 live Actually make changes
813
814Options:
815 --verbose, -v Show detailed progress
816 --user=PATTERN Only sync users matching pattern
817 --anonymous-only Only sync anonymous content (skip all users)
818 --wipe=HANDLE Wipe all ATProto data for user (e.g., --wipe=jeffrey.at.aesthetic.computer)
819 --restore=HANDLE Restore ATProto data for user from MongoDB (e.g., --restore=jeffrey.at.aesthetic.computer)
820 --types=TYPE1,TYPE2 Only sync specific types (paintings,moods,pieces,kidlisp,tapes)
821 --paintings-only Only sync paintings
822 --moods-only Only sync moods
823 --pieces-only Only sync pieces
824 --kidlisp-only Only sync kidlisp
825 --tapes-only Only sync tapes
826 --help, -h Show this help
827
828Examples:
829 node sync-atproto.mjs # Dry run all content types
830 node sync-atproto.mjs live # Sync all content types
831 node sync-atproto.mjs live --paintings-only # Only sync paintings
832 node sync-atproto.mjs live --anonymous-only --tapes-only # Only sync anonymous tapes
833 node sync-atproto.mjs live --user=jeffrey # Only sync jeffrey's content
834 node sync-atproto.mjs live --types=paintings,moods # Only paintings and moods
835 node sync-atproto.mjs dry-run --verbose # See detailed dry run
836
837 # Wipe examples
838 node sync-atproto.mjs --wipe=jeffrey.at.aesthetic.computer # Dry run wipe
839 node sync-atproto.mjs live --wipe=jeffrey.at.aesthetic.computer # Wipe jeffrey's ATProto data
840
841 # Restore examples
842 node sync-atproto.mjs --restore=jeffrey.at.aesthetic.computer # Dry run restore
843 node sync-atproto.mjs live --restore=jeffrey.at.aesthetic.computer # Restore all content
844 node sync-atproto.mjs live --restore=jeffrey.at.aesthetic.computer --paintings-only # Only restore paintings
845 node sync-atproto.mjs live --restore=jeffrey.at.aesthetic.computer --types=paintings,moods # Only paintings and moods
846`);
847 process.exit(0);
848}
849
850// Handle wipe mode
851if (wipeUser) {
852 wipeUserAtprotoData(wipeUser, { dryRun, contentTypes })
853 .then(() => {
854 console.log(`\n✅ Done!\n`);
855 process.exit(0);
856 })
857 .catch(error => {
858 console.error('\n❌ Fatal error:', error);
859 process.exit(1);
860 });
861} else if (restoreUser) {
862 // Handle restore mode
863 restoreUserAtprotoData(restoreUser, { dryRun, contentTypes })
864 .then(() => {
865 console.log(`\n✅ Done!\n`);
866 process.exit(0);
867 })
868 .catch(error => {
869 console.error('\n❌ Fatal error:', error);
870 process.exit(1);
871 });
872} else {
873 syncAll({ dryRun, verbose, userFilter, contentTypes })
874 .then(() => {
875 console.log(`\n✅ Done!\n`);
876 process.exit(0);
877 })
878 .catch(error => {
879 console.error('\n❌ Fatal error:', error);
880 process.exit(1);
881 });
882}