testing local-first datastores
1import { mkdir, writeFile } from 'fs/promises';
2import { join } from 'path';
3import { generateUrls } from './urls.js';
4import { generateImages } from './images.js';
5import { generateDocuments } from './documents.js';
6import { generateMetadata } from './metadata.js';
7
8const OUTPUT_DIR = 'test-data';
9
10const COUNTS = {
11 urls: 10000,
12 images: 1000,
13 documents: 1000,
14 metadata: 100000
15};
16
17function formatBytes(bytes: number): string {
18 if (bytes < 1024) return `${bytes} B`;
19 if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
20 return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
21}
22
23function progressBar(current: number, total: number, label: string): void {
24 const width = 30;
25 const percent = current / total;
26 const filled = Math.floor(width * percent);
27 const bar = '█'.repeat(filled) + '░'.repeat(width - filled);
28 process.stdout.write(`\r${label}: [${bar}] ${current}/${total}`);
29 if (current === total) console.log();
30}
31
32async function main() {
33 console.log('Generating test data...\n');
34 await mkdir(OUTPUT_DIR, { recursive: true });
35
36 // Generate URLs
37 console.log(`Generating ${COUNTS.urls} URLs...`);
38 const urls = generateUrls(COUNTS.urls);
39 const urlsJson = JSON.stringify(urls, null, 2);
40 await writeFile(join(OUTPUT_DIR, 'urls.json'), urlsJson);
41 console.log(` URLs: ${formatBytes(urlsJson.length)}\n`);
42
43 // Generate metadata
44 console.log(`Generating ${COUNTS.metadata} metadata rows...`);
45 const metadata = generateMetadata(COUNTS.metadata);
46 const metadataJson = JSON.stringify(metadata, null, 2);
47 await writeFile(join(OUTPUT_DIR, 'metadata.json'), metadataJson);
48 console.log(` Metadata: ${formatBytes(metadataJson.length)}\n`);
49
50 // Generate images (slow, show progress)
51 console.log(`Generating ${COUNTS.images} images...`);
52 const images = await generateImages(COUNTS.images, OUTPUT_DIR, (current, total) => {
53 progressBar(current, total, 'Images');
54 });
55 const totalImageSize = images.reduce((sum, img) => sum + img.size, 0);
56 await writeFile(join(OUTPUT_DIR, 'images.json'), JSON.stringify(images, null, 2));
57 console.log(` Total image size: ${formatBytes(totalImageSize)}\n`);
58
59 // Generate documents (show progress)
60 console.log(`Generating ${COUNTS.documents} documents...`);
61 const documents = await generateDocuments(COUNTS.documents, OUTPUT_DIR, (current, total) => {
62 progressBar(current, total, 'Documents');
63 });
64 const totalDocSize = documents.reduce((sum, doc) => sum + doc.size, 0);
65 await writeFile(join(OUTPUT_DIR, 'documents.json'), JSON.stringify(documents, null, 2));
66 console.log(` Total document size: ${formatBytes(totalDocSize)}\n`);
67
68 // Summary
69 const totalSize = urlsJson.length + metadataJson.length + totalImageSize + totalDocSize;
70 console.log('='.repeat(50));
71 console.log('Generation complete!');
72 console.log(`Total data size: ${formatBytes(totalSize)}`);
73 console.log(`Output directory: ${OUTPUT_DIR}/`);
74}
75
76main().catch(console.error);