testing local-first datastores
1import { chromium, firefox, webkit, Browser } from '@playwright/test';
2import { createServer, Server } from 'http';
3import { readFileSync, writeFileSync, mkdirSync } from 'fs';
4import { dirname, join } from 'path';
5import { fileURLToPath } from 'url';
6import * as vega from 'vega';
7import * as vegaLite from 'vega-lite';
8import sharp from 'sharp';
9import type { BrowserStoreBenchmarkResults, BrowserBenchmarkResult } from '../harness/browser-types.js';
10
11const __filename = fileURLToPath(import.meta.url);
12const __dirname = dirname(__filename);
13
14const HARNESS_PATH = join(__dirname, 'harness.html');
15const BROWSERS = ['chromium', 'firefox', 'webkit'] as const;
16const STORES = ['indexeddb', 'localstorage', 'pouchdb', 'pglite'] as const;
17const ITERATIONS = 10;
18const PORT = 9876;
19
20type BrowserType = typeof BROWSERS[number];
21type StoreType = typeof STORES[number];
22
23interface StoreRunResult {
24 success: boolean;
25 error?: string;
26 init: BrowserBenchmarkResult;
27 writes: {
28 allUrls: BrowserBenchmarkResult;
29 allMetadata: BrowserBenchmarkResult;
30 allImages: BrowserBenchmarkResult;
31 allDocuments: BrowserBenchmarkResult;
32 };
33 reads: {
34 recentUrls: BrowserBenchmarkResult;
35 randomImages: BrowserBenchmarkResult;
36 randomDocuments: BrowserBenchmarkResult;
37 };
38 storage: { totalBytes: number };
39}
40
41interface BrowserRunResults {
42 browserName: string;
43 userAgent: string;
44 stores: Record<StoreType, BrowserStoreBenchmarkResults>;
45 success: boolean;
46 error?: string;
47}
48
49function formatDuration(ms: number): string {
50 if (ms < 1000) return `${ms.toFixed(0)}ms`;
51 if (ms < 60000) return `${(ms / 1000).toFixed(2)}s`;
52 return `${(ms / 60000).toFixed(2)}m`;
53}
54
55function medianExcludingExtremes(values: number[]): number {
56 if (values.length <= 2) return values[0] ?? 0;
57 const sorted = [...values].sort((a, b) => a - b);
58 const trimmed = sorted.slice(1, -1);
59 const mid = Math.floor(trimmed.length / 2);
60 if (trimmed.length % 2 === 0) {
61 return (trimmed[mid - 1] + trimmed[mid]) / 2;
62 }
63 return trimmed[mid];
64}
65
66function aggregateBenchmarkResults(results: BrowserBenchmarkResult[]): BrowserBenchmarkResult {
67 const successful = results.filter(r => !r.failed);
68 if (successful.length === 0) return results[0];
69
70 const durations = successful.map(r => r.durationMs);
71 return {
72 name: successful[0].name,
73 durationMs: medianExcludingExtremes(durations),
74 itemCount: successful[0].itemCount,
75 bytesProcessed: successful[0].bytesProcessed,
76 failed: false
77 };
78}
79
80function aggregateStoreRuns(runs: StoreRunResult[], storeName: string, userAgent: string): BrowserStoreBenchmarkResults {
81 const successful = runs.filter(r => r.success);
82 if (successful.length === 0) {
83 return {
84 storeName,
85 userAgent,
86 init: { name: 'init', durationMs: 0, failed: true, error: runs[0]?.error },
87 writes: {
88 allUrls: { name: 'allUrls', durationMs: 0, failed: true },
89 allMetadata: { name: 'allMetadata', durationMs: 0, failed: true },
90 allImages: { name: 'allImages', durationMs: 0, failed: true },
91 allDocuments: { name: 'allDocuments', durationMs: 0, failed: true }
92 },
93 reads: {
94 recentUrls: { name: 'recentUrls', durationMs: 0, failed: true },
95 randomImages: { name: 'randomImages', durationMs: 0, failed: true },
96 randomDocuments: { name: 'randomDocuments', durationMs: 0, failed: true }
97 },
98 storage: { totalBytes: 0 }
99 };
100 }
101
102 return {
103 storeName,
104 userAgent,
105 init: aggregateBenchmarkResults(successful.map(r => r.init)),
106 writes: {
107 allUrls: aggregateBenchmarkResults(successful.map(r => r.writes.allUrls)),
108 allMetadata: aggregateBenchmarkResults(successful.map(r => r.writes.allMetadata)),
109 allImages: aggregateBenchmarkResults(successful.map(r => r.writes.allImages)),
110 allDocuments: aggregateBenchmarkResults(successful.map(r => r.writes.allDocuments))
111 },
112 reads: {
113 recentUrls: aggregateBenchmarkResults(successful.map(r => r.reads.recentUrls)),
114 randomImages: aggregateBenchmarkResults(successful.map(r => r.reads.randomImages)),
115 randomDocuments: aggregateBenchmarkResults(successful.map(r => r.reads.randomDocuments))
116 },
117 storage: {
118 totalBytes: medianExcludingExtremes(successful.map(r => r.storage.totalBytes))
119 }
120 };
121}
122
123const BENCHMARK_TIMEOUT = 30000; // 30 seconds per benchmark
124
125async function benchmarkStore(page: any, store: StoreType): Promise<StoreRunResult> {
126 const script = `
127 (async function(storeName) {
128 const testData = window.generateTestData();
129
130 function createResult(name, durationMs, itemCount, bytesProcessed) {
131 return { name: name, durationMs: durationMs, itemCount: itemCount, bytesProcessed: bytesProcessed, failed: false };
132 }
133
134 if (storeName === 'indexeddb') {
135 const DB_NAME = 'localstress-bench';
136
137 try {
138 // Delete old DB
139 await new Promise(function(resolve, reject) {
140 const req = indexedDB.deleteDatabase(DB_NAME);
141 req.onsuccess = function() { resolve(); };
142 req.onerror = function() { reject(req.error); };
143 });
144
145 // Init
146 const initStart = performance.now();
147 const db = await new Promise(function(resolve, reject) {
148 const req = indexedDB.open(DB_NAME, 1);
149 req.onupgradeneeded = function(e) {
150 const db = e.target.result;
151 const urlStore = db.createObjectStore('urls', { keyPath: 'id' });
152 urlStore.createIndex('createdAt', 'createdAt');
153 db.createObjectStore('images', { keyPath: 'id' });
154 db.createObjectStore('documents', { keyPath: 'id' });
155 const metaStore = db.createObjectStore('metadata', { keyPath: 'id' });
156 metaStore.createIndex('timestamp', 'timestamp');
157 };
158 req.onsuccess = function() { resolve(req.result); };
159 req.onerror = function() { reject(req.error); };
160 });
161 const initDuration = performance.now() - initStart;
162
163 // Write URLs
164 const urlsStart = performance.now();
165 await new Promise(function(resolve, reject) {
166 const tx = db.transaction(['urls'], 'readwrite');
167 const store = tx.objectStore('urls');
168 for (let i = 0; i < testData.urls.length; i++) {
169 store.add(testData.urls[i]);
170 }
171 tx.oncomplete = function() { resolve(); };
172 tx.onerror = function() { reject(tx.error); };
173 });
174 const urlsDuration = performance.now() - urlsStart;
175
176 // Write Metadata
177 const metaStart = performance.now();
178 await new Promise(function(resolve, reject) {
179 const tx = db.transaction(['metadata'], 'readwrite');
180 const store = tx.objectStore('metadata');
181 for (let i = 0; i < testData.metadata.length; i++) {
182 store.add(testData.metadata[i]);
183 }
184 tx.oncomplete = function() { resolve(); };
185 tx.onerror = function() { reject(tx.error); };
186 });
187 const metaDuration = performance.now() - metaStart;
188
189 // Write Images
190 const imagesStart = performance.now();
191 let imageBytes = 0;
192 await new Promise(function(resolve, reject) {
193 const tx = db.transaction(['images'], 'readwrite');
194 const store = tx.objectStore('images');
195 for (let i = 0; i < testData.images.length; i++) {
196 const img = testData.images[i];
197 store.add({ id: img.id, data: img.data, size: img.data.byteLength });
198 imageBytes += img.data.byteLength;
199 }
200 tx.oncomplete = function() { resolve(); };
201 tx.onerror = function() { reject(tx.error); };
202 });
203 const imagesDuration = performance.now() - imagesStart;
204
205 // Write Documents
206 const docsStart = performance.now();
207 let docBytes = 0;
208 await new Promise(function(resolve, reject) {
209 const tx = db.transaction(['documents'], 'readwrite');
210 const store = tx.objectStore('documents');
211 for (let i = 0; i < testData.documents.length; i++) {
212 const doc = testData.documents[i];
213 const size = new TextEncoder().encode(doc.content).byteLength;
214 store.add({ id: doc.id, content: doc.content, size: size });
215 docBytes += size;
216 }
217 tx.oncomplete = function() { resolve(); };
218 tx.onerror = function() { reject(tx.error); };
219 });
220 const docsDuration = performance.now() - docsStart;
221
222 // Read recent URLs
223 const readUrlsStart = performance.now();
224 await new Promise(function(resolve, reject) {
225 const tx = db.transaction(['urls'], 'readonly');
226 const idx = tx.objectStore('urls').index('createdAt');
227 const req = idx.openCursor(null, 'prev');
228 const results = [];
229 req.onsuccess = function(e) {
230 const cursor = e.target.result;
231 if (cursor && results.length < 100) {
232 results.push(cursor.value);
233 cursor.continue();
234 } else {
235 resolve(results);
236 }
237 };
238 req.onerror = function() { reject(req.error); };
239 });
240 const readUrlsDuration = performance.now() - readUrlsStart;
241
242 // Read random images
243 const readImagesStart = performance.now();
244 const imageIds = testData.images.slice(0, Math.min(10, testData.images.length)).map(function(i) { return i.id; });
245 let readImageBytes = 0;
246 if (imageIds.length > 0) {
247 await new Promise(function(resolve, reject) {
248 const tx = db.transaction(['images'], 'readonly');
249 const store = tx.objectStore('images');
250 let completed = 0;
251 for (let i = 0; i < imageIds.length; i++) {
252 const req = store.get(imageIds[i]);
253 req.onsuccess = function() {
254 if (req.result) readImageBytes += req.result.data.byteLength;
255 completed++;
256 if (completed === imageIds.length) resolve();
257 };
258 req.onerror = function() { reject(req.error); };
259 }
260 });
261 }
262 const readImagesDuration = performance.now() - readImagesStart;
263
264 // Read random documents
265 const readDocsStart = performance.now();
266 const docIds = testData.documents.slice(0, Math.min(50, testData.documents.length)).map(function(d) { return d.id; });
267 let readDocBytes = 0;
268 if (docIds.length > 0) {
269 await new Promise(function(resolve, reject) {
270 const tx = db.transaction(['documents'], 'readonly');
271 const store = tx.objectStore('documents');
272 let completed = 0;
273 for (let i = 0; i < docIds.length; i++) {
274 const req = store.get(docIds[i]);
275 req.onsuccess = function() {
276 if (req.result) readDocBytes += new TextEncoder().encode(req.result.content).byteLength;
277 completed++;
278 if (completed === docIds.length) resolve();
279 };
280 req.onerror = function() { reject(req.error); };
281 }
282 });
283 }
284 const readDocsDuration = performance.now() - readDocsStart;
285
286 // Storage
287 const estimate = await navigator.storage.estimate();
288 const storageBytes = estimate.usage || 0;
289
290 db.close();
291 await new Promise(function(resolve) {
292 const req = indexedDB.deleteDatabase(DB_NAME);
293 req.onsuccess = function() { resolve(); };
294 req.onerror = function() { resolve(); };
295 });
296
297 return {
298 success: true,
299 init: createResult('init', initDuration),
300 writes: {
301 allUrls: createResult('allUrls', urlsDuration, testData.urls.length),
302 allMetadata: createResult('allMetadata', metaDuration, testData.metadata.length),
303 allImages: createResult('allImages', imagesDuration, testData.images.length, imageBytes),
304 allDocuments: createResult('allDocuments', docsDuration, testData.documents.length, docBytes)
305 },
306 reads: {
307 recentUrls: createResult('recentUrls', readUrlsDuration, 100),
308 randomImages: createResult('randomImages', readImagesDuration, imageIds.length, readImageBytes),
309 randomDocuments: createResult('randomDocuments', readDocsDuration, docIds.length, readDocBytes)
310 },
311 storage: { totalBytes: storageBytes }
312 };
313 } catch (error) {
314 return { success: false, error: error.message };
315 }
316
317 } else if (storeName === 'localstorage') {
318 try {
319 localStorage.clear();
320
321 // Init
322 const initStart = performance.now();
323 const initDuration = performance.now() - initStart;
324
325 // Write URLs
326 const urlsStart = performance.now();
327 localStorage.setItem('urls', JSON.stringify(testData.urls));
328 const urlsDuration = performance.now() - urlsStart;
329
330 // Write Metadata
331 const metaStart = performance.now();
332 localStorage.setItem('metadata', JSON.stringify(testData.metadata));
333 const metaDuration = performance.now() - metaStart;
334
335 // Images - base64 encode
336 const imagesStart = performance.now();
337 let imageBytes = 0;
338 const imageData = [];
339 for (let i = 0; i < testData.images.length; i++) {
340 const img = testData.images[i];
341 const base64 = btoa(String.fromCharCode.apply(null, img.data));
342 imageData.push({ id: img.id, data: base64 });
343 imageBytes += img.data.byteLength;
344 }
345 localStorage.setItem('images', JSON.stringify(imageData));
346 const imagesDuration = performance.now() - imagesStart;
347
348 // Documents
349 const docsStart = performance.now();
350 let docBytes = 0;
351 for (let i = 0; i < testData.documents.length; i++) {
352 docBytes += new TextEncoder().encode(testData.documents[i].content).byteLength;
353 }
354 localStorage.setItem('documents', JSON.stringify(testData.documents));
355 const docsDuration = performance.now() - docsStart;
356
357 // Read URLs
358 const readUrlsStart = performance.now();
359 const urls = JSON.parse(localStorage.getItem('urls') || '[]');
360 urls.sort(function(a, b) { return b.createdAt - a.createdAt; }).slice(0, 100);
361 const readUrlsDuration = performance.now() - readUrlsStart;
362
363 // Read Images
364 const readImagesStart = performance.now();
365 const images = JSON.parse(localStorage.getItem('images') || '[]');
366 let readImageBytes = 0;
367 for (let i = 0; i < Math.min(10, images.length); i++) {
368 const binary = atob(images[i].data);
369 readImageBytes += binary.length;
370 }
371 const readImagesDuration = performance.now() - readImagesStart;
372
373 // Read Documents
374 const readDocsStart = performance.now();
375 const docs = JSON.parse(localStorage.getItem('documents') || '[]');
376 let readDocBytes = 0;
377 for (let i = 0; i < Math.min(50, docs.length); i++) {
378 readDocBytes += new TextEncoder().encode(docs[i].content).byteLength;
379 }
380 const readDocsDuration = performance.now() - readDocsStart;
381
382 // Storage estimate
383 const estimate = await navigator.storage.estimate();
384 const storageBytes = estimate.usage || 0;
385
386 localStorage.clear();
387
388 return {
389 success: true,
390 init: createResult('init', initDuration),
391 writes: {
392 allUrls: createResult('allUrls', urlsDuration, testData.urls.length),
393 allMetadata: createResult('allMetadata', metaDuration, testData.metadata.length),
394 allImages: createResult('allImages', imagesDuration, testData.images.length, imageBytes),
395 allDocuments: createResult('allDocuments', docsDuration, testData.documents.length, docBytes)
396 },
397 reads: {
398 recentUrls: createResult('recentUrls', readUrlsDuration, 100),
399 randomImages: createResult('randomImages', readImagesDuration, 10, readImageBytes),
400 randomDocuments: createResult('randomDocuments', readDocsDuration, 50, readDocBytes)
401 },
402 storage: { totalBytes: storageBytes }
403 };
404 } catch (error) {
405 return { success: false, error: error.message };
406 }
407
408 } else if (storeName === 'pouchdb') {
409 if (!window.PouchDB) {
410 return { success: false, error: 'PouchDB not loaded' };
411 }
412
413 const PouchDB = window.PouchDB;
414 const DB_PREFIX = 'localstress-pouchdb-' + Date.now();
415
416 try {
417 // Init - create databases
418 const initStart = performance.now();
419 const urlsDb = new PouchDB(DB_PREFIX + '-urls');
420 const metaDb = new PouchDB(DB_PREFIX + '-metadata');
421 const imagesDb = new PouchDB(DB_PREFIX + '-images');
422 const docsDb = new PouchDB(DB_PREFIX + '-documents');
423 const initDuration = performance.now() - initStart;
424
425 // Write URLs
426 const urlsStart = performance.now();
427 const urlDocs = testData.urls.map(function(url) {
428 return { _id: url.id, url: url.url, title: url.title, createdAt: url.createdAt, tags: url.tags };
429 });
430 await urlsDb.bulkDocs(urlDocs);
431 const urlsDuration = performance.now() - urlsStart;
432
433 // Write Metadata
434 const metaStart = performance.now();
435 const metaDocs = testData.metadata.map(function(m) {
436 return { _id: m.id, key: m.key, value: m.value, category: m.category, timestamp: m.timestamp };
437 });
438 await metaDb.bulkDocs(metaDocs);
439 const metaDuration = performance.now() - metaStart;
440
441 // Write Images (as base64)
442 const imagesStart = performance.now();
443 let imageBytes = 0;
444 for (let i = 0; i < testData.images.length; i++) {
445 const img = testData.images[i];
446 // Convert Uint8Array to base64 in chunks to avoid stack overflow
447 let binary = '';
448 const bytes = img.data;
449 const chunkSize = 8192;
450 for (let j = 0; j < bytes.length; j += chunkSize) {
451 const chunk = bytes.subarray(j, j + chunkSize);
452 binary += String.fromCharCode.apply(null, chunk);
453 }
454 const base64 = btoa(binary);
455 await imagesDb.put({ _id: img.id, data: base64, size: img.data.byteLength });
456 imageBytes += img.data.byteLength;
457 }
458 const imagesDuration = performance.now() - imagesStart;
459
460 // Write Documents
461 const docsStart = performance.now();
462 let docBytes = 0;
463 const docDocs = testData.documents.map(function(doc) {
464 const size = new TextEncoder().encode(doc.content).byteLength;
465 docBytes += size;
466 return { _id: doc.id, content: doc.content, size: size };
467 });
468 await docsDb.bulkDocs(docDocs);
469 const docsDuration = performance.now() - docsStart;
470
471 // Read recent URLs - PouchDB allDocs with descending
472 const readUrlsStart = performance.now();
473 const urlResult = await urlsDb.allDocs({ include_docs: true, limit: 100 });
474 // Sort by createdAt descending (PouchDB doesn't have native sorting without views)
475 const sortedUrls = urlResult.rows
476 .map(function(r) { return r.doc; })
477 .sort(function(a, b) { return b.createdAt - a.createdAt; })
478 .slice(0, 100);
479 const readUrlsDuration = performance.now() - readUrlsStart;
480
481 // Read random images
482 const readImagesStart = performance.now();
483 const imageIds = testData.images.slice(0, Math.min(10, testData.images.length)).map(function(i) { return i.id; });
484 let readImageBytes = 0;
485 for (let i = 0; i < imageIds.length; i++) {
486 try {
487 const doc = await imagesDb.get(imageIds[i]);
488 if (doc.data) {
489 const binary = atob(doc.data);
490 readImageBytes += binary.length;
491 }
492 } catch (e) {}
493 }
494 const readImagesDuration = performance.now() - readImagesStart;
495
496 // Read random documents
497 const readDocsStart = performance.now();
498 const docIds = testData.documents.slice(0, Math.min(50, testData.documents.length)).map(function(d) { return d.id; });
499 let readDocBytes = 0;
500 for (let i = 0; i < docIds.length; i++) {
501 try {
502 const doc = await docsDb.get(docIds[i]);
503 if (doc.content) {
504 readDocBytes += new TextEncoder().encode(doc.content).byteLength;
505 }
506 } catch (e) {}
507 }
508 const readDocsDuration = performance.now() - readDocsStart;
509
510 // Storage
511 const estimate = await navigator.storage.estimate();
512 const storageBytes = estimate.usage || 0;
513
514 // Cleanup
515 await urlsDb.destroy();
516 await metaDb.destroy();
517 await imagesDb.destroy();
518 await docsDb.destroy();
519
520 return {
521 success: true,
522 init: createResult('init', initDuration),
523 writes: {
524 allUrls: createResult('allUrls', urlsDuration, testData.urls.length),
525 allMetadata: createResult('allMetadata', metaDuration, testData.metadata.length),
526 allImages: createResult('allImages', imagesDuration, testData.images.length, imageBytes),
527 allDocuments: createResult('allDocuments', docsDuration, testData.documents.length, docBytes)
528 },
529 reads: {
530 recentUrls: createResult('recentUrls', readUrlsDuration, 100),
531 randomImages: createResult('randomImages', readImagesDuration, imageIds.length, readImageBytes),
532 randomDocuments: createResult('randomDocuments', readDocsDuration, docIds.length, readDocBytes)
533 },
534 storage: { totalBytes: storageBytes }
535 };
536 } catch (error) {
537 return { success: false, error: error.message };
538 }
539
540 } else if (storeName === 'pglite') {
541 if (!window.PGlite) {
542 return { success: false, error: 'PGlite not loaded' };
543 }
544
545 const PGlite = window.PGlite;
546
547 try {
548 // Init - create in-memory database and tables
549 const initStart = performance.now();
550 const db = new PGlite();
551 await db.exec(\`
552 CREATE TABLE IF NOT EXISTS urls (
553 id TEXT PRIMARY KEY,
554 url TEXT,
555 title TEXT,
556 created_at BIGINT,
557 tags TEXT
558 );
559 CREATE TABLE IF NOT EXISTS metadata (
560 id TEXT PRIMARY KEY,
561 key TEXT,
562 value TEXT,
563 category TEXT,
564 timestamp BIGINT
565 );
566 CREATE TABLE IF NOT EXISTS images (
567 id TEXT PRIMARY KEY,
568 data TEXT,
569 size INTEGER
570 );
571 CREATE TABLE IF NOT EXISTS documents (
572 id TEXT PRIMARY KEY,
573 content TEXT,
574 size INTEGER
575 );
576 CREATE INDEX IF NOT EXISTS idx_urls_created ON urls(created_at);
577 CREATE INDEX IF NOT EXISTS idx_meta_timestamp ON metadata(timestamp);
578 \`);
579 const initDuration = performance.now() - initStart;
580
581 // Write URLs
582 const urlsStart = performance.now();
583 for (let i = 0; i < testData.urls.length; i++) {
584 const u = testData.urls[i];
585 await db.query(
586 'INSERT INTO urls (id, url, title, created_at, tags) VALUES ($1, $2, $3, $4, $5)',
587 [u.id, u.url, u.title, u.createdAt, JSON.stringify(u.tags)]
588 );
589 }
590 const urlsDuration = performance.now() - urlsStart;
591
592 // Write Metadata
593 const metaStart = performance.now();
594 for (let i = 0; i < testData.metadata.length; i++) {
595 const m = testData.metadata[i];
596 await db.query(
597 'INSERT INTO metadata (id, key, value, category, timestamp) VALUES ($1, $2, $3, $4, $5)',
598 [m.id, m.key, String(m.value), m.category, m.timestamp]
599 );
600 }
601 const metaDuration = performance.now() - metaStart;
602
603 // Write Images (as base64)
604 const imagesStart = performance.now();
605 let imageBytes = 0;
606 for (let i = 0; i < testData.images.length; i++) {
607 const img = testData.images[i];
608 let binary = '';
609 const bytes = img.data;
610 const chunkSize = 8192;
611 for (let j = 0; j < bytes.length; j += chunkSize) {
612 const chunk = bytes.subarray(j, j + chunkSize);
613 binary += String.fromCharCode.apply(null, chunk);
614 }
615 const base64 = btoa(binary);
616 await db.query(
617 'INSERT INTO images (id, data, size) VALUES ($1, $2, $3)',
618 [img.id, base64, img.data.byteLength]
619 );
620 imageBytes += img.data.byteLength;
621 }
622 const imagesDuration = performance.now() - imagesStart;
623
624 // Write Documents
625 const docsStart = performance.now();
626 let docBytes = 0;
627 for (let i = 0; i < testData.documents.length; i++) {
628 const doc = testData.documents[i];
629 const size = new TextEncoder().encode(doc.content).byteLength;
630 await db.query(
631 'INSERT INTO documents (id, content, size) VALUES ($1, $2, $3)',
632 [doc.id, doc.content, size]
633 );
634 docBytes += size;
635 }
636 const docsDuration = performance.now() - docsStart;
637
638 // Read recent URLs
639 const readUrlsStart = performance.now();
640 await db.query('SELECT * FROM urls ORDER BY created_at DESC LIMIT 100');
641 const readUrlsDuration = performance.now() - readUrlsStart;
642
643 // Read random images
644 const readImagesStart = performance.now();
645 const imageIds = testData.images.slice(0, Math.min(10, testData.images.length)).map(function(i) { return i.id; });
646 let readImageBytes = 0;
647 for (let i = 0; i < imageIds.length; i++) {
648 const result = await db.query('SELECT data FROM images WHERE id = $1', [imageIds[i]]);
649 if (result.rows.length > 0 && result.rows[0].data) {
650 const binary = atob(result.rows[0].data);
651 readImageBytes += binary.length;
652 }
653 }
654 const readImagesDuration = performance.now() - readImagesStart;
655
656 // Read random documents
657 const readDocsStart = performance.now();
658 const docIds = testData.documents.slice(0, Math.min(50, testData.documents.length)).map(function(d) { return d.id; });
659 let readDocBytes = 0;
660 for (let i = 0; i < docIds.length; i++) {
661 const result = await db.query('SELECT content FROM documents WHERE id = $1', [docIds[i]]);
662 if (result.rows.length > 0 && result.rows[0].content) {
663 readDocBytes += new TextEncoder().encode(result.rows[0].content).byteLength;
664 }
665 }
666 const readDocsDuration = performance.now() - readDocsStart;
667
668 // Storage estimate
669 const estimate = await navigator.storage.estimate();
670 const storageBytes = estimate.usage || 0;
671
672 // Cleanup
673 await db.close();
674
675 return {
676 success: true,
677 init: createResult('init', initDuration),
678 writes: {
679 allUrls: createResult('allUrls', urlsDuration, testData.urls.length),
680 allMetadata: createResult('allMetadata', metaDuration, testData.metadata.length),
681 allImages: createResult('allImages', imagesDuration, testData.images.length, imageBytes),
682 allDocuments: createResult('allDocuments', docsDuration, testData.documents.length, docBytes)
683 },
684 reads: {
685 recentUrls: createResult('recentUrls', readUrlsDuration, 100),
686 randomImages: createResult('randomImages', readImagesDuration, imageIds.length, readImageBytes),
687 randomDocuments: createResult('randomDocuments', readDocsDuration, docIds.length, readDocBytes)
688 },
689 storage: { totalBytes: storageBytes }
690 };
691 } catch (error) {
692 return { success: false, error: error.message };
693 }
694 }
695
696 return { success: false, error: 'Unknown store' };
697 })('${store}')
698 `;
699
700 try {
701 const result = await Promise.race([
702 page.evaluate(script),
703 new Promise<StoreRunResult>((_, reject) =>
704 setTimeout(() => reject(new Error('Benchmark timeout')), BENCHMARK_TIMEOUT)
705 )
706 ]);
707 return result as StoreRunResult;
708 } catch (error) {
709 const err = error as Error;
710 return {
711 success: false,
712 error: err.message,
713 init: { name: 'init', durationMs: 0, failed: true, error: err.message },
714 writes: {
715 allUrls: { name: 'allUrls', durationMs: 0, failed: true },
716 allMetadata: { name: 'allMetadata', durationMs: 0, failed: true },
717 allImages: { name: 'allImages', durationMs: 0, failed: true },
718 allDocuments: { name: 'allDocuments', durationMs: 0, failed: true }
719 },
720 reads: {
721 recentUrls: { name: 'recentUrls', durationMs: 0, failed: true },
722 randomImages: { name: 'randomImages', durationMs: 0, failed: true },
723 randomDocuments: { name: 'randomDocuments', durationMs: 0, failed: true }
724 },
725 storage: { totalBytes: 0 }
726 };
727 }
728}
729
730async function runBrowserBenchmarks(
731 browserType: BrowserType,
732 serverUrl: string
733): Promise<BrowserRunResults> {
734 console.log(`\n${'─'.repeat(60)}`);
735 console.log(`Testing ${browserType.toUpperCase()} (${ITERATIONS} iterations per store)`);
736 console.log('─'.repeat(60));
737
738 let browser: Browser | undefined;
739 const result: BrowserRunResults = {
740 browserName: browserType,
741 userAgent: '',
742 stores: {} as Record<StoreType, BrowserStoreBenchmarkResults>,
743 success: false
744 };
745
746 try {
747 if (browserType === 'chromium') {
748 browser = await chromium.launch({ headless: true });
749 } else if (browserType === 'firefox') {
750 browser = await firefox.launch({ headless: true });
751 } else if (browserType === 'webkit') {
752 browser = await webkit.launch({ headless: true });
753 }
754
755 const page = await browser!.newPage();
756
757 console.log(' Loading harness...');
758 await page.goto(serverUrl, { waitUntil: 'domcontentloaded' });
759
760 await page.waitForFunction(() => (window as any).generateTestData !== undefined, {
761 timeout: 10000
762 });
763
764 result.userAgent = await page.evaluate(() => navigator.userAgent);
765
766 for (const store of STORES) {
767 console.log(`\n Benchmarking ${store.toUpperCase()}`);
768 const runs: StoreRunResult[] = [];
769 let consecutiveFailures = 0;
770
771 for (let i = 0; i < ITERATIONS; i++) {
772 process.stdout.write(` [${i + 1}/${ITERATIONS}] `);
773 const runResult = await benchmarkStore(page, store);
774 runs.push(runResult);
775
776 if (runResult.success) {
777 consecutiveFailures = 0;
778 console.log(`✓ ${formatDuration(runResult.init.durationMs + runResult.writes.allUrls.durationMs + runResult.writes.allMetadata.durationMs)}`);
779 } else {
780 consecutiveFailures++;
781 console.log(`✗ ${runResult.error}`);
782 // If 3 consecutive failures (including timeouts), skip remaining iterations
783 if (consecutiveFailures >= 3) {
784 console.log(` Skipping remaining iterations due to repeated failures`);
785 break;
786 }
787 }
788 }
789
790 result.stores[store] = aggregateStoreRuns(runs, store, result.userAgent);
791 }
792
793 await page.close();
794 result.success = true;
795
796 } catch (error) {
797 const err = error as Error;
798 result.error = err.message;
799
800 if (err.message.includes("Executable doesn't exist")) {
801 console.error(` ✗ Browser not installed. Run: npm run browsers:install`);
802 } else {
803 console.error(` ✗ Error: ${err.message}`);
804 }
805 } finally {
806 if (browser) {
807 try {
808 await browser.close();
809 } catch {
810 // Ignore
811 }
812 }
813 }
814
815 return result;
816}
817
818function startServer(): Promise<{ url: string; server: Server }> {
819 const harness = readFileSync(HARNESS_PATH, 'utf-8');
820
821 return new Promise((resolve) => {
822 const server = createServer((req, res) => {
823 if (req.url === '/') {
824 res.writeHead(200, { 'Content-Type': 'text/html' });
825 res.end(harness);
826 } else {
827 res.writeHead(404);
828 res.end('Not found');
829 }
830 });
831
832 server.listen(PORT, () => {
833 resolve({ url: `http://localhost:${PORT}`, server });
834 });
835 });
836}
837
838async function main() {
839 console.log('='.repeat(60));
840 console.log('BROWSER DATASTORE BENCHMARK SUITE');
841 console.log('='.repeat(60));
842 console.log(`Iterations per store: ${ITERATIONS} (excluding high/low, using median)`);
843
844 const { url: serverUrl, server } = await startServer();
845 console.log(`Server started at ${serverUrl}`);
846
847 const allResults: BrowserRunResults[] = [];
848
849 for (const browserType of BROWSERS) {
850 const result = await runBrowserBenchmarks(browserType, serverUrl);
851 allResults.push(result);
852
853 if (result.success) {
854 console.log(`\n ✓ ${browserType} completed`);
855 } else {
856 console.log(`\n ✗ ${browserType} failed: ${result.error}`);
857 }
858 }
859
860 server.close();
861
862 // Save results
863 const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
864 const runDir = join('results', `browser-benchmark-${timestamp}`);
865 mkdirSync(runDir, { recursive: true });
866
867 const resultsData = {
868 timestamp: new Date().toISOString(),
869 type: 'browser',
870 iterations: ITERATIONS,
871 browsers: allResults.map(r => ({
872 browserName: r.browserName,
873 userAgent: r.userAgent,
874 success: r.success,
875 error: r.error,
876 stores: r.stores
877 }))
878 };
879
880 writeFileSync(join(runDir, 'results.json'), JSON.stringify(resultsData, null, 2));
881 console.log(`\nResults saved to ${runDir}/results.json`);
882
883 // Print summary
884 console.log(`\n${'='.repeat(60)}`);
885 console.log('BROWSER BENCHMARK SUMMARY');
886 console.log('='.repeat(60));
887
888 for (const browserResult of allResults) {
889 if (!browserResult.success) continue;
890
891 console.log(`\n${browserResult.browserName.toUpperCase()}`);
892 for (const [storeName, storeResult] of Object.entries(browserResult.stores)) {
893 if (storeResult.init.failed) {
894 console.log(` ${storeName}: FAILED`);
895 continue;
896 }
897
898 const totalWrite = storeResult.writes.allUrls.durationMs +
899 storeResult.writes.allMetadata.durationMs +
900 storeResult.writes.allImages.durationMs +
901 storeResult.writes.allDocuments.durationMs;
902
903 const totalRead = storeResult.reads.recentUrls.durationMs +
904 storeResult.reads.randomImages.durationMs +
905 storeResult.reads.randomDocuments.durationMs;
906
907 console.log(` ${storeName}: init=${formatDuration(storeResult.init.durationMs)} write=${formatDuration(totalWrite)} read=${formatDuration(totalRead)}`);
908 }
909 }
910
911 const successCount = allResults.filter(r => r.success).length;
912 console.log(`\nBrowsers tested: ${successCount}/${BROWSERS.length} successful`);
913
914 // Generate charts and HTML report
915 if (successCount > 0) {
916 console.log('\nGenerating charts...');
917 await generateBrowserCharts(allResults, runDir);
918 console.log(`Charts and report saved to ${runDir}/`);
919 }
920}
921
922const COLORS = ['#3498db', '#e74c3c', '#2ecc71', '#f39c12', '#9b59b6'];
923const WIDTH = 700;
924const HEIGHT = 400;
925
926async function renderVegaLiteChart(spec: any): Promise<Buffer> {
927 const vegaSpec = vegaLite.compile(spec).spec;
928 const view = new vega.View(vega.parse(vegaSpec), { renderer: 'none' });
929 const svg = await view.toSVG();
930 return await sharp(Buffer.from(svg)).png().toBuffer();
931}
932
933async function generateBrowserCharts(results: BrowserRunResults[], outputDir: string): Promise<void> {
934 const successfulResults = results.filter(r => r.success);
935 if (successfulResults.length === 0) return;
936
937 // Prepare data for charts - one bar per browser/store combo
938 const writeData: any[] = [];
939 const readData: any[] = [];
940
941 for (const browserResult of successfulResults) {
942 for (const [storeName, storeResult] of Object.entries(browserResult.stores)) {
943 if (storeResult.init.failed) continue;
944
945 const label = `${browserResult.browserName}/${storeName}`;
946
947 const totalWrite = storeResult.writes.allUrls.durationMs +
948 storeResult.writes.allMetadata.durationMs +
949 storeResult.writes.allImages.durationMs +
950 storeResult.writes.allDocuments.durationMs;
951
952 const totalRead = storeResult.reads.recentUrls.durationMs +
953 storeResult.reads.randomImages.durationMs +
954 storeResult.reads.randomDocuments.durationMs;
955
956 writeData.push({ label, browser: browserResult.browserName, store: storeName, value: totalWrite });
957 readData.push({ label, browser: browserResult.browserName, store: storeName, value: totalRead });
958 }
959 }
960
961 // Write performance chart
962 const writeSpec = {
963 $schema: 'https://vega.github.io/schema/vega-lite/v5.json',
964 width: WIDTH,
965 height: HEIGHT,
966 title: { text: 'Browser Write Performance', fontSize: 18 },
967 data: { values: writeData },
968 mark: 'bar',
969 encoding: {
970 x: {
971 field: 'label',
972 type: 'nominal',
973 title: null,
974 axis: { labelAngle: -45, labelFontSize: 10 },
975 sort: null
976 },
977 y: {
978 field: 'value',
979 type: 'quantitative',
980 title: 'Duration (ms)',
981 axis: { titleFontSize: 12, labelFontSize: 11 }
982 },
983 color: {
984 field: 'store',
985 type: 'nominal',
986 title: 'Store',
987 scale: { range: COLORS },
988 legend: { titleFontSize: 12, labelFontSize: 11 }
989 }
990 },
991 config: { background: 'white', view: { stroke: null } }
992 };
993
994 const writeChart = await renderVegaLiteChart(writeSpec);
995 writeFileSync(join(outputDir, 'write-comparison.png'), writeChart);
996
997 // Read performance chart
998 const readSpec = {
999 $schema: 'https://vega.github.io/schema/vega-lite/v5.json',
1000 width: WIDTH,
1001 height: HEIGHT,
1002 title: { text: 'Browser Read Performance', fontSize: 18 },
1003 data: { values: readData },
1004 mark: 'bar',
1005 encoding: {
1006 x: {
1007 field: 'label',
1008 type: 'nominal',
1009 title: null,
1010 axis: { labelAngle: -45, labelFontSize: 10 },
1011 sort: null
1012 },
1013 y: {
1014 field: 'value',
1015 type: 'quantitative',
1016 title: 'Duration (ms)',
1017 axis: { titleFontSize: 12, labelFontSize: 11 }
1018 },
1019 color: {
1020 field: 'store',
1021 type: 'nominal',
1022 title: 'Store',
1023 scale: { range: COLORS },
1024 legend: { titleFontSize: 12, labelFontSize: 11 }
1025 }
1026 },
1027 config: { background: 'white', view: { stroke: null } }
1028 };
1029
1030 const readChart = await renderVegaLiteChart(readSpec);
1031 writeFileSync(join(outputDir, 'read-comparison.png'), readChart);
1032
1033 // Generate HTML report
1034 const html = generateBrowserHtmlReport(successfulResults);
1035 writeFileSync(join(outputDir, 'report.html'), html);
1036}
1037
1038function generateBrowserHtmlReport(results: BrowserRunResults[]): string {
1039 const storeNames = [...new Set(results.flatMap(r => Object.keys(r.stores)))];
1040
1041 // Calculate totals for each browser/store combo
1042 type CellData = { write: number; read: number; failed: boolean };
1043 const cellData: Map<string, CellData> = new Map();
1044
1045 for (const browserResult of results) {
1046 for (const storeName of storeNames) {
1047 const key = `${browserResult.browserName}-${storeName}`;
1048 const storeResult = browserResult.stores[storeName as StoreType];
1049
1050 if (!storeResult || storeResult.init.failed) {
1051 cellData.set(key, { write: Infinity, read: Infinity, failed: true });
1052 } else {
1053 const totalWrite = storeResult.writes.allUrls.durationMs +
1054 storeResult.writes.allMetadata.durationMs +
1055 storeResult.writes.allImages.durationMs +
1056 storeResult.writes.allDocuments.durationMs;
1057 const totalRead = storeResult.reads.recentUrls.durationMs +
1058 storeResult.reads.randomImages.durationMs +
1059 storeResult.reads.randomDocuments.durationMs;
1060 cellData.set(key, { write: totalWrite, read: totalRead, failed: false });
1061 }
1062 }
1063 }
1064
1065 // Find min/max for each store column (excluding failed)
1066 const storeStats: Map<string, { minWrite: number; maxWrite: number; minRead: number; maxRead: number }> = new Map();
1067 for (const storeName of storeNames) {
1068 const values = results
1069 .map(br => cellData.get(`${br.browserName}-${storeName}`))
1070 .filter(v => v && !v.failed) as CellData[];
1071
1072 if (values.length > 0) {
1073 storeStats.set(storeName, {
1074 minWrite: Math.min(...values.map(v => v.write)),
1075 maxWrite: Math.max(...values.map(v => v.write)),
1076 minRead: Math.min(...values.map(v => v.read)),
1077 maxRead: Math.max(...values.map(v => v.read))
1078 });
1079 }
1080 }
1081
1082 // Build table rows with color coding
1083 const tableRows = results.map(browserResult => {
1084 const cells = storeNames.map(storeName => {
1085 const key = `${browserResult.browserName}-${storeName}`;
1086 const data = cellData.get(key);
1087 const stats = storeStats.get(storeName);
1088
1089 if (!data || data.failed) {
1090 return '<td class="failed">FAILED</td>';
1091 }
1092
1093 // Determine colors (green = best/lowest, red = worst/highest)
1094 let writeClass = '';
1095 let readClass = '';
1096
1097 if (stats) {
1098 if (data.write === stats.minWrite) writeClass = 'best';
1099 else if (data.write === stats.maxWrite) writeClass = 'worst';
1100
1101 if (data.read === stats.minRead) readClass = 'best';
1102 else if (data.read === stats.maxRead) readClass = 'worst';
1103 }
1104
1105 const writeSpan = writeClass
1106 ? `<span class="${writeClass}">${formatDuration(data.write)}</span>`
1107 : formatDuration(data.write);
1108 const readSpan = readClass
1109 ? `<span class="${readClass}">${formatDuration(data.read)}</span>`
1110 : formatDuration(data.read);
1111
1112 return `<td>${writeSpan} / ${readSpan}</td>`;
1113 });
1114
1115 return `
1116 <tr>
1117 <td><strong>${browserResult.browserName}</strong></td>
1118 ${cells.join('\n ')}
1119 </tr>`;
1120 }).join('\n');
1121
1122 return `<!DOCTYPE html>
1123<html lang="en">
1124<head>
1125 <meta charset="UTF-8">
1126 <meta name="viewport" content="width=device-width, initial-scale=1.0">
1127 <title>Browser Benchmark Results</title>
1128 <style>
1129 body {
1130 font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
1131 max-width: 1200px;
1132 margin: 0 auto;
1133 padding: 20px;
1134 background: #f5f5f5;
1135 }
1136 h1 { color: #2c3e50; text-align: center; }
1137 .timestamp { text-align: center; color: #7f8c8d; margin-bottom: 30px; }
1138 .container {
1139 background: white;
1140 border-radius: 8px;
1141 padding: 20px;
1142 margin-bottom: 20px;
1143 box-shadow: 0 2px 4px rgba(0,0,0,0.1);
1144 }
1145 table { width: 100%; border-collapse: collapse; font-size: 14px; }
1146 th, td { padding: 12px 8px; text-align: center; border-bottom: 1px solid #eee; }
1147 th { background: #f8f9fa; font-weight: 600; color: #2c3e50; }
1148 td:first-child { text-align: left; }
1149 .failed { color: #e74c3c; font-weight: 600; }
1150 .best { color: #27ae60; font-weight: 600; }
1151 .worst { color: #e74c3c; font-weight: 600; }
1152 .charts { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; }
1153 .chart { background: white; border-radius: 8px; padding: 20px; box-shadow: 0 2px 4px rgba(0,0,0,0.1); text-align: center; }
1154 .chart img { max-width: 100%; height: auto; }
1155 .info { background: #e3f2fd; padding: 15px; border-radius: 4px; margin-bottom: 20px; }
1156 .info p { margin: 5px 0; font-size: 14px; }
1157 @media (max-width: 900px) { .charts { grid-template-columns: 1fr; } }
1158 </style>
1159</head>
1160<body>
1161 <h1>Browser Datastore Benchmark Results</h1>
1162 <p class="timestamp">${new Date().toLocaleString()}</p>
1163
1164 <div class="container">
1165 <div class="info">
1166 <p><strong>Environment:</strong> Browser-based benchmarks running in Chromium, Firefox, and WebKit</p>
1167 <p><strong>Methodology:</strong> 10 iterations per store, excluding high/low, reporting median</p>
1168 <p><strong>Test Data:</strong> 1000 URLs, 10000 metadata rows, 20 images (50KB each = 1MB), 500 documents</p>
1169 <p><strong>Values shown:</strong> Total Write Time / Total Read Time</p>
1170 <p><strong>Note:</strong> PouchDB fails on WebKit in Playwright due to a compatibility issue with PouchDB's IndexedDB adapter (raw IndexedDB works fine)</p>
1171 </div>
1172
1173 <h2>Summary</h2>
1174 <table>
1175 <thead>
1176 <tr>
1177 <th>Browser</th>
1178 ${storeNames.map(s => `<th>${s}</th>`).join('\n ')}
1179 </tr>
1180 </thead>
1181 <tbody>
1182 ${tableRows}
1183 </tbody>
1184 </table>
1185 </div>
1186
1187 <div class="charts">
1188 <div class="chart">
1189 <img src="write-comparison.png" alt="Write Performance">
1190 </div>
1191 <div class="chart">
1192 <img src="read-comparison.png" alt="Read Performance">
1193 </div>
1194 </div>
1195</body>
1196</html>`;
1197}
1198
1199main().catch((err) => {
1200 console.error('Benchmark failed:', err);
1201 process.exit(1);
1202});