A public mirror for the whole atmosphere
hubble.microcosm.blue
1<!DOCTYPE html>
2<html lang="en">
3<head>
4<meta charset="utf-8">
5<title>rocks space efficiency</title>
6<script src="https://cdn.plot.ly/plotly-2.35.2.min.js"></script>
7<style>
8 * { box-sizing: border-box; margin: 0; padding: 0; }
9 body { font-family: system-ui, -apple-system, sans-serif; background: #f5f5f5; color: #222; padding: 20px; }
10 h1 { margin-bottom: 10px; }
11 .subtitle { color: #666; margin-bottom: 20px; }
12 .chart-container { background: #fff; border-radius: 8px; padding: 16px; margin-bottom: 20px; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
13 .chart-container h2 { font-size: 16px; margin-bottom: 8px; }
14 .chart-row { display: flex; gap: 20px; flex-wrap: wrap; }
15 .chart-row > .chart-container { flex: 1; min-width: 500px; }
16 table { border-collapse: collapse; width: 100%; font-size: 13px; }
17 th, td { text-align: right; padding: 4px 10px; border-bottom: 1px solid #e0e0e0; }
18 th { background: #f0f0f0; cursor: pointer; user-select: none; position: sticky; top: 0; }
19 th:hover { background: #e0e0e0; }
20 td:first-child, th:first-child { text-align: left; }
21 tr:hover td { background: #f8f8ff; }
22 .table-wrap { max-height: 500px; overflow-y: auto; }
23 .filters { display: flex; gap: 16px; flex-wrap: wrap; margin-bottom: 16px; align-items: center; }
24 .filters label { font-size: 13px; font-weight: 600; }
25 .filters select { font-size: 13px; padding: 2px 6px; }
26 .zero-toggle { float: right; font-size: 12px; font-weight: normal; color: #666; cursor: pointer; user-select: none; }
27 .zero-toggle input { vertical-align: middle; cursor: pointer; }
28 .dl-btn { float: right; font-size: 12px; padding: 2px 10px; cursor: pointer; border: 1px solid #ccc; border-radius: 4px; background: #f8f8f8; color: #444; }
29 .dl-btn:hover { background: #e8e8e8; }
30</style>
31</head>
32<body>
33
34<h1>rocks records space efficiency</h1>
35<p class="subtitle">atproto records: <span id="total-keys"></span> keys across <span id="total-accounts"></span> accounts. <span id="config-count"></span> rocksdb configs measured in <span id="total-time"></span>.</p>
36
37<div class="chart-row">
38 <div class="chart-container">
39 <h2>value size distribution</h2>
40 <div id="chart-value-hist"></div>
41 </div>
42 <div class="chart-container">
43 <h2>keys per account distribution</h2>
44 <div id="chart-account-hist"></div>
45 </div>
46</div>
47
48<div class="chart-row">
49 <div class="chart-container">
50 <h2>total size vs block size by zstd level & dictionary <label class="zero-toggle"><input type="checkbox" data-chart="chart-block-size"> y from 0</label></h2>
51 <div id="chart-block-size"></div>
52 </div>
53 <div class="chart-container">
54 <h2>total size vs compaction time <label class="zero-toggle"><input type="checkbox" data-chart="chart-pareto"> y from 0</label></h2>
55 <div id="chart-pareto"></div>
56 </div>
57</div>
58
59<div class="chart-row">
60 <div class="chart-container">
61 <h2>dictionary training by block size <label class="zero-toggle"><input type="checkbox" data-chart="chart-dict"> y from 0</label></h2>
62 <div id="chart-dict"></div>
63 </div>
64 <div class="chart-container">
65 <h2>dictionary benefit vs block size <label class="zero-toggle"><input type="checkbox" data-chart="chart-dict-benefit"> y from 0</label></h2>
66 <div id="chart-dict-benefit"></div>
67 </div>
68</div>
69
70<div class="chart-row">
71 <div class="chart-container">
72 <h2>dictionary: size reduction vs compaction time <label class="zero-toggle"><input type="checkbox" data-chart="chart-dict-cost"> y from 0</label></h2>
73 <div id="chart-dict-cost"></div>
74 </div>
75 <div class="chart-container">
76 <h2>32k blocks: zstd level × dictionary <label class="zero-toggle"><input type="checkbox" data-chart="chart-32k"> y from 0</label></h2>
77 <div id="chart-32k"></div>
78 </div>
79</div>
80
81<div class="chart-row">
82 <div class="chart-container">
83 <h2>key restart interval <label class="zero-toggle"><input type="checkbox" data-chart="chart-restart"> y from 0</label></h2>
84 <div id="chart-restart"></div>
85 </div>
86</div>
87
88<div class="chart-row">
89 <div class="chart-container">
90 <h2>subcompactions: size <label class="zero-toggle"><input type="checkbox" data-chart="chart-subcomp-size"> y from 0</label></h2>
91 <div id="chart-subcomp-size"></div>
92 </div>
93 <div class="chart-container">
94 <h2>subcompactions: time <label class="zero-toggle"><input type="checkbox" data-chart="chart-subcomp-time"> y from 0</label></h2>
95 <div id="chart-subcomp-time"></div>
96 </div>
97</div>
98
99<div class="chart-container">
100 <h2>all data <button class="dl-btn" id="download-csv">download csv</button></h2>
101 <div class="filters">
102 <label>block size: <select id="f-block"><option value="">all</option></select></label>
103 <label>zstd level: <select id="f-level"><option value="">all</option></select></label>
104 <label>dict bytes: <select id="f-dict"><option value="">all</option></select></label>
105 </div>
106 <div class="table-wrap">
107 <table id="results-table">
108 <thead><tr>
109 <th data-col="block_size">block_size</th>
110 <th data-col="zstd_level">zstd_level</th>
111 <th data-col="restart_interval">restart_int</th>
112 <th data-col="dict_bytes">dict_bytes</th>
113 <th data-col="train_mult">train_mult</th>
114 <th data-col="opt_filters">filters</th>
115 <th data-col="subcompactions">subcomp</th>
116 <th data-col="size_mib">size_gib</th>
117 <th data-col="compact_secs">compact_s</th>
118 </tr></thead>
119 <tbody id="results-body"></tbody>
120 </table>
121 </div>
122</div>
123
124<script>
125// ── embed CSV data ────────────────────────────────────────────────
126const csvText = `block_size,zstd_level,restart_interval,dict_bytes,train_mult,opt_filters,subcompactions,size_bytes,size_mib,compact_secs
1274096,0,8,0,,true,10,31395311799,29940,19.8
1284096,1,8,0,,true,10,22116416091,21091,30.7
1294096,1,8,16384,16,true,10,20659329178,19702,42.5
1304096,3,4,0,,true,10,22374836755,21338,51.4
1314096,3,8,0,,true,10,22100643559,21076,41.7
1324096,3,8,1024,,true,10,21177970088,20196,57.0
1334096,3,8,1024,4,true,10,22100668085,21076,42.2
1344096,3,8,1024,8,true,10,22100676471,21076,43.2
1354096,3,8,1024,16,true,10,22100669290,21076,42.9
1364096,3,8,1024,32,true,10,21526349249,20529,55.1
1374096,3,8,1024,64,true,10,21501054074,20505,60.0
1384096,3,8,2048,,true,10,21184609495,20203,59.8
1394096,3,8,2048,4,true,10,22100673247,21076,42.1
1404096,3,8,2048,8,true,10,22100621663,21076,41.2
1414096,3,8,2048,16,true,10,20635793718,19679,52.0
1424096,3,8,2048,32,true,10,20562444599,19609,58.2
1434096,3,8,2048,64,true,10,20508962586,19558,57.6
1444096,3,8,4096,,true,10,21128768715,20149,61.0
1454096,3,8,4096,4,true,10,22100685331,21076,41.4
1464096,3,8,4096,8,true,10,20577552594,19624,51.9
1474096,3,8,4096,16,true,10,20479709404,19530,60.2
1484096,3,8,4096,32,true,10,20403721320,19458,60.9
1494096,3,8,4096,64,true,10,20368652511,19425,62.7
1504096,3,8,8192,,true,10,21057872538,20082,62.9
1514096,3,8,8192,4,true,10,20632375521,19676,54.1
1524096,3,8,8192,8,true,10,20506735960,19556,61.2
1534096,3,8,8192,16,true,10,20390435178,19445,62.1
1544096,3,8,8192,32,true,10,20363765340,19420,61.9
1554096,3,8,8192,64,true,10,20331974936,19390,61.6
1564096,3,8,16384,,true,10,21083190741,20106,65.0
1574096,3,8,16384,4,true,10,20579253867,19625,54.2
1584096,3,8,16384,8,true,10,20455471802,19507,61.6
1594096,3,8,16384,16,true,10,20405638509,19460,63.1
1604096,3,8,16384,32,true,10,20386866978,19442,65.3
1614096,3,8,16384,64,true,10,20367953764,19424,67.5
1624096,3,8,65536,,true,10,20893557686,19925,63.9
1634096,3,8,65536,4,true,10,20409555824,19464,56.4
1644096,3,8,65536,8,true,10,20345337995,19402,64.1
1654096,3,8,65536,16,true,10,20298445731,19358,65.1
1664096,3,8,65536,32,true,10,20261884271,19323,69.0
1674096,3,8,65536,64,true,10,20239324109,19301,74.3
1684096,3,16,0,,true,10,21909626509,20894,48.0
1694096,3,32,0,,true,10,21823083686,20812,39.5
1704096,3,64,0,,true,10,21823091636,20812,40.4
1714096,6,8,0,,true,10,21750720790,20743,65.2
1724096,6,8,16384,16,true,10,20146634422,19213,84.1
1734096,9,8,0,,true,10,21746294515,20738,111.8
1744096,9,8,16384,16,true,10,19849302063,18929,116.6
1758192,0,8,0,,true,10,29379196929,28018,35.0
1768192,1,8,0,,true,10,20005967746,19079,25.5
1778192,1,8,16384,16,true,10,19572338449,18665,34.0
1788192,3,8,0,,false,10,20031898121,19103,36.1
1798192,3,8,0,,true,10,20031920603,19103,31.9
1808192,3,8,1024,,true,10,19622953913,18713,48.9
1818192,3,8,1024,4,true,10,20031917101,19103,33.8
1828192,3,8,1024,8,true,10,20031918966,19103,33.5
1838192,3,8,1024,16,true,10,20031904492,19103,33.5
1848192,3,8,1024,32,true,10,20031915783,19103,33.1
1858192,3,8,1024,64,true,10,20182475006,19247,50.1
1868192,3,8,2048,,true,10,19620130060,18711,52.0
1878192,3,8,2048,4,true,10,20031954949,19103,35.7
1888192,3,8,2048,8,true,10,20031939452,19103,33.7
1898192,3,8,2048,16,true,10,20031949661,19103,33.4
1908192,3,8,2048,32,true,10,20143626622,19210,47.8
1918192,3,8,2048,64,true,10,20120486239,19188,52.3
1928192,3,8,4096,,true,10,19635342683,18725,48.9
1938192,3,8,4096,4,true,10,20031954812,19103,33.2
1948192,3,8,4096,8,true,10,20031942177,19103,33.3
1958192,3,8,4096,16,true,10,19372492949,18475,47.6
1968192,3,8,4096,32,true,10,19313920339,18419,50.4
1978192,3,8,4096,64,true,10,19282505585,18389,52.1
1988192,3,8,8192,,true,10,19635635166,18726,50.9
1998192,3,8,8192,4,true,10,20031928773,19103,35.0
2008192,3,8,8192,8,true,10,19410703858,18511,46.3
2018192,3,8,8192,16,true,10,19341160203,18445,50.9
2028192,3,8,8192,32,true,10,19296132462,18402,50.9
2038192,3,8,8192,64,true,10,19273631354,18380,51.8
2048192,3,8,16384,,true,10,19655701449,18745,54.3
2058192,3,8,16384,4,true,10,19502525484,18599,50.5
2068192,3,8,16384,8,true,10,19419249737,18519,51.3
2078192,3,8,16384,16,true,10,19358437498,18461,52.4
2088192,3,8,16384,32,true,10,19340825956,18444,53.1
2098192,3,8,16384,64,true,10,19329911907,18434,54.7
2108192,3,8,65536,,true,10,19654121838,18743,52.6
2118192,3,8,65536,4,true,10,19436643569,18536,50.1
2128192,3,8,65536,8,true,10,19376580371,18478,54.5
2138192,3,8,65536,16,true,10,19337019622,18441,55.6
2148192,3,8,65536,32,true,10,19318086405,18423,59.0
2158192,3,8,65536,64,true,10,19305582440,18411,64.8
2168192,6,8,0,,true,10,19702719743,18789,62.2
2178192,6,8,16384,16,true,10,19065986792,18182,74.2
2188192,9,8,0,,true,10,19693493308,18781,100.9
2198192,9,8,16384,16,true,10,18747260904,17878,100.7
22016384,0,8,0,,true,10,28042902316,26743,26.0
22116384,1,8,0,,true,10,18751959286,17883,22.9
22216384,1,8,16384,16,true,10,18714866778,17847,30.2
22316384,3,8,0,,true,1,18829928785,17957,174.2
22416384,3,8,0,,true,4,18829973330,17957,44.1
22516384,3,8,0,,true,10,18830038815,17957,27.8
22616384,3,8,16384,16,true,10,18615498021,17753,43.5
22716384,6,8,0,,true,10,18431835527,17577,57.9
22816384,6,8,16384,16,true,10,18267813872,17421,69.3
22916384,9,8,0,,true,10,18408979927,17556,109.1
23016384,9,8,16384,16,true,10,17956014400,17124,95.6
23132768,0,8,0,,true,10,27149041457,25891,22.3
23232768,1,8,0,,true,10,18270567659,17424,21.6
23332768,1,8,16384,16,true,10,18211125411,17367,27.5
23432768,3,4,0,,true,10,18324784539,17475,28.5
23532768,3,8,0,,false,10,18101433642,17262,27.5
23632768,3,8,0,,true,10,18101394905,17262,26.5
23732768,3,8,16384,16,true,10,18076847819,17239,46.5
23832768,3,16,0,,true,10,17971384889,17138,28.1
23932768,3,32,0,,true,10,17900644184,17071,27.1
24032768,3,64,0,,true,10,17859565644,17032,29.2
24132768,6,8,0,,true,10,17875945091,17047,56.9
24232768,6,8,16384,16,true,10,17743235923,16921,72.5
24332768,9,8,0,,true,10,17672322648,16853,76.0
24432768,9,8,16384,16,true,10,17455398843,16646,96.0
24565536,0,8,0,,true,10,26622152522,25388,21.2
24665536,1,8,0,,true,10,17829479045,17003,21.0
24765536,1,8,16384,16,true,10,17829619661,17003,21.4
24865536,3,8,0,,true,10,17676395890,16857,25.5
24965536,3,8,16384,16,true,10,17676353730,16857,26.4
25065536,6,8,0,,true,10,17445895583,16637,53.6
25165536,6,8,16384,16,true,10,17445851139,16637,54.6
25265536,9,8,0,,true,10,17228573735,16430,73.1
25365536,9,8,16384,16,true,10,17228513145,16430,73.9
254131072,0,8,0,,true,10,26362266962,25141,19.5
255131072,1,8,0,,true,10,17595617600,16780,20.4
256131072,1,8,16384,16,true,10,17595914003,16780,20.0
257131072,3,8,0,,true,10,17381327820,16576,24.9
258131072,3,8,16384,16,true,10,17381302914,16576,25.5
259131072,6,8,0,,true,10,17164905178,16369,54.9
260131072,6,8,16384,16,true,10,17165045658,16369,56.9
261131072,9,8,0,,true,10,16940090482,16155,76.5
262131072,9,8,16384,16,true,10,16940006516,16155,76.7
26316384,3,8,16384,64,true,10,18575595180,17715,46.8
26416384,3,8,65536,4,true,10,18720555347,17853,49.6
26516384,3,8,65536,16,true,10,18636700580,17773,52.1
26616384,3,8,262144,1,true,10,20116140943,19184,62.1
26716384,3,8,262144,4,true,10,18718065045,17850,62.1
26832768,3,8,16384,64,true,10,18037187627,17201,55.5
26932768,3,8,65536,4,true,10,18160440220,17319,61.1
27032768,3,8,65536,16,true,10,18108588895,17269,56.7
27132768,3,8,262144,1,true,10,18702653174,17836,59.6
27232768,3,8,262144,4,true,10,18174889606,17332,68.0
27365536,3,8,16384,64,true,10,17666766140,16848,48.3
27465536,3,8,65536,4,true,10,17676764811,16857,27.2
27565536,3,8,65536,16,true,10,17718370930,16897,51.0
27665536,3,8,262144,1,true,10,17676607265,16857,27.5
27765536,3,8,262144,4,true,10,17806456807,16981,65.2
27832768,1,8,16384,,true,10,18145390797,17304,28.8
27932768,1,8,16384,64,true,10,18162942166,17321,28.7
28032768,1,8,65536,,true,10,18113630667,17274,25.4
28132768,1,8,65536,4,true,10,18252364775,17406,25.5
28232768,1,8,65536,16,true,10,18191268324,17348,27.3
28332768,1,8,262144,,true,10,18433074184,17579,25.8
28432768,1,8,262144,1,true,10,18941076049,18063,30.4
28532768,1,8,262144,4,true,10,18630362351,17767,32.4
28632768,3,8,16384,,true,10,18092158457,17254,43.3
28732768,3,8,65536,,true,10,18175762988,17333,50.1
28832768,3,8,262144,,true,10,18237799128,17392,58.5
28932768,6,8,16384,,true,10,17815254359,16989,67.7
29032768,6,8,16384,64,true,10,17700896109,16880,71.0
29132768,6,8,65536,,true,10,17903486595,17074,74.4
29232768,6,8,65536,4,true,10,17835381202,17009,75.6
29332768,6,8,65536,16,true,10,17792150430,16967,79.5
29432768,6,8,262144,,true,10,17617765307,16801,82.0
29532768,6,8,262144,1,true,10,17722853142,16901,85.7
29632768,6,8,262144,4,true,10,17519172262,16707,89.4
29732768,9,8,16384,,true,10,17542440248,16729,93.6
29832768,9,8,16384,64,true,10,17402812286,16596,96.1
29932768,9,8,65536,,true,10,17621780999,16805,103.5
30032768,9,8,65536,4,true,10,17560386951,16746,106.3
30132768,9,8,65536,16,true,10,17513473140,16702,110.7
30232768,9,8,262144,,true,10,17288349711,16487,114.8
30332768,9,8,262144,1,true,10,17736615600,16914,134.2
30432768,9,8,262144,4,true,10,17193958194,16397,126.5
30532768,6,8,16384,128,true,10,17685606257,16866,73.3
30632768,6,8,32768,,true,10,17851433056,17024,69.4
30732768,6,8,32768,16,true,10,17756182902,16933,72.2
30832768,6,8,32768,64,true,10,17701788554,16881,77.0
30932768,6,8,32768,128,true,10,17677204659,16858,80.8
31032768,9,8,16384,128,true,10,17377098336,16572,100.3
31132768,9,8,32768,,true,10,17573743202,16759,97.5
31232768,9,8,32768,16,true,10,17464617794,16655,101.3
31332768,9,8,32768,64,true,10,17397585233,16591,109.2
31432768,9,8,32768,128,true,10,17363679655,16559,114.3
315`;
316
317// ── parse ─────────────────────────────────────────────────────────
318function parseCSV(text) {
319 const lines = text.trim().split('\n');
320 const headers = lines[0].split(',');
321 return lines.slice(1).map(line => {
322 const vals = line.split(',');
323 const row = {};
324 headers.forEach((h, i) => {
325 const v = vals[i];
326 if (h === 'opt_filters') row[h] = v === 'true';
327 else if (h === 'train_mult') row[h] = v === '' ? null : +v;
328 else row[h] = +v;
329 });
330 return row;
331 });
332}
333
334const data = parseCSV(csvText);
335document.getElementById('config-count').textContent = data.length;
336const totalSecs = data.reduce((s, d) => s + d.compact_secs, 0);
337const totalMins = (totalSecs / 60).toFixed(1);
338document.getElementById('total-time').textContent = `${totalMins}min`;
339
340// ── data profile stats ───────────────────────────────────────────
341const stats = {
342 "total_keys": 220118913,
343 "total_accounts": 155536,
344 "value_size_hist": [
345 [16, 32, 62],
346 [32, 64, 59915],
347 [64, 128, 24326635],
348 [128, 256, 160003952],
349 [256, 512, 27257670],
350 [512, 1024, 7251053],
351 [1024, 2048, 1189568],
352 [2048, 4096, 28324],
353 [4096, 8192, 1495],
354 [8192, 16384, 188],
355 [16384, 32768, 37],
356 [32768, 65536, 11],
357 [65536, 131072, 2],
358 [131072, 262144, 1]
359 ],
360 "keys_per_account_hist": [
361 [1, 2, 5192],
362 [2, 4, 16553],
363 [4, 8, 13913],
364 [8, 16, 13736],
365 [16, 32, 15006],
366 [32, 64, 15720],
367 [64, 128, 15330],
368 [128, 256, 14137],
369 [256, 512, 12703],
370 [512, 1024, 10464],
371 [1024, 2048, 8027],
372 [2048, 4096, 5631],
373 [4096, 8192, 3899],
374 [8192, 16384, 2536],
375 [16384, 32768, 1463],
376 [32768, 65536, 789],
377 [65536, 131072, 322],
378 [131072, 262144, 92],
379 [262144, 524288, 20],
380 [524288, 1048576, 3]
381 ]
382};
383
384const fmtNum = n => n.toLocaleString();
385document.getElementById('total-keys').textContent = fmtNum(stats.total_keys);
386document.getElementById('total-accounts').textContent = fmtNum(stats.total_accounts);
387
388// ── helpers ───────────────────────────────────────────────────────
389const gib = d => +(d.size_mib / 1024).toFixed(2);
390const levelName = l => l === 0 ? 'lz4' : `z${l}`;
391const levelColors = { 0: '#888', 1: '#4ecdc4', 3: '#45b7d1', 6: '#f7b731', 9: '#fc5c65' };
392
393const plotLayout = (opts = {}) => ({
394 margin: { t: 30, b: 50, l: 60, r: 20 },
395 hovermode: 'closest',
396 font: { family: 'system-ui', size: 12 },
397 ...opts,
398});
399
400function groupBy(arr, keyFn) {
401 const groups = {};
402 arr.forEach(d => {
403 const k = keyFn(d);
404 (groups[k] = groups[k] || []).push(d);
405 });
406 return groups;
407}
408
409// ── chart 1: size vs block size (with level slider) ──────────────
410function chartBlockSize() {
411 // filter to standard configs: ri=8, filters=on, subcomp=10
412 const base = data.filter(d =>
413 d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10 &&
414 ((d.dict_bytes === 0 && d.train_mult === null) ||
415 (d.dict_bytes === 16384 && d.train_mult === 16))
416 );
417
418 const levels = [...new Set(base.map(d => d.zstd_level))].sort((a, b) => a - b);
419
420 // build all traces (one per level × dict combo)
421 const traces = [];
422 const traceLevel = []; // which level each trace belongs to
423 for (const level of levels) {
424 for (const dict of [0, 16384]) {
425 const subset = base.filter(d => d.zstd_level === level && d.dict_bytes === dict);
426 if (subset.length === 0) continue;
427 subset.sort((a, b) => a.block_size - b.block_size);
428 traces.push({
429 x: subset.map(d => d.block_size / 1024 + 'k'),
430 y: subset.map(gib),
431 name: `${levelName(level)} ${dict > 0 ? 'dict=16k' : 'no dict'}`,
432 mode: 'lines+markers',
433 line: { dash: dict > 0 ? 'dash' : 'solid', color: levelColors[level] || '#888', width: 2.5 },
434 marker: { symbol: dict > 0 ? 'diamond' : 'circle', size: 8 },
435 text: subset.map(d => `${d.compact_secs}s`),
436 hovertemplate: '%{x} block, ' + levelName(level) + (dict > 0 ? ' +dict' : '') +
437 '<br>%{y:.2f} GiB<br>%{text}<extra></extra>',
438 });
439 traceLevel.push(level);
440 }
441 }
442
443 // per-level steps: highlight one, fade the rest
444 const steps = [{
445 label: 'all',
446 method: 'restyle',
447 args: [{
448 'line.color': traceLevel.map(l => levelColors[l] || '#888'),
449 'line.width': traceLevel.map(() => 2.5),
450 'marker.opacity': traceLevel.map(() => 1),
451 'marker.size': traceLevel.map(() => 8),
452 }],
453 }];
454
455 for (const hl of levels) {
456 steps.push({
457 label: levelName(hl),
458 method: 'restyle',
459 args: [{
460 'line.color': traceLevel.map(l => l === hl ? levelColors[l] : (levelColors[l] || '#888') + '30'),
461 'line.width': traceLevel.map(l => l === hl ? 3.5 : 1),
462 'marker.opacity': traceLevel.map(l => l === hl ? 1 : 0.2),
463 'marker.size': traceLevel.map(l => l === hl ? 10 : 5),
464 }],
465 });
466 }
467
468 Plotly.newPlot('chart-block-size', traces, plotLayout({
469 xaxis: { title: 'Block Size', type: 'category' },
470 yaxis: { title: 'Size (GiB)' },
471 legend: { x: 1, xanchor: 'right', y: 1 },
472 sliders: [{
473 active: 0,
474 pad: { t: 30 },
475 currentvalue: { prefix: 'zstd level: ', font: { size: 13 } },
476 steps: steps,
477 }],
478 }));
479}
480
481// ── chart 2: pareto (size vs time, with zstd level slider) ───────
482function chartPareto() {
483 const blockColors = {
484 4096: '#fc5c65', 8192: '#f7b731', 16384: '#45b7d1',
485 32768: '#4ecdc4', 65536: '#a55eea',
486 };
487 const blocks = [...new Set(data.map(d => d.block_size))].sort((a, b) => a - b);
488 const levels = [...new Set(data.map(d => d.zstd_level))].sort((a, b) => a - b);
489
490 const traces = [];
491 const traceLevel = [];
492
493 for (const bs of blocks) {
494 const subset = data.filter(d => d.block_size === bs && d.subcompactions === 10);
495 traces.push({
496 x: subset.map(d => d.compact_secs),
497 y: subset.map(gib),
498 name: (bs / 1024) + 'k',
499 mode: 'markers',
500 marker: { size: 7, color: subset.map(() => blockColors[bs] || '#888'), opacity: 1 },
501 customdata: subset.map(d => d.zstd_level),
502 text: subset.map(d => {
503 const parts = [levelName(d.zstd_level), `ri=${d.restart_interval}`];
504 if (d.dict_bytes > 0) parts.push(`dict=${d.dict_bytes}`);
505 if (d.train_mult !== null) parts.push(`train=${d.train_mult}x`);
506 return parts.join(', ');
507 }),
508 hovertemplate: '%{text}<br>%{y:.2f} GiB, %{x:.1f}s<extra>%{fullData.name} blocks</extra>',
509 });
510 traceLevel.push(subset.map(d => d.zstd_level));
511 }
512
513 // slider: highlight by zstd level
514 const steps = [{
515 label: 'all',
516 method: 'restyle',
517 args: [{
518 'marker.opacity': traceLevel.map(levels => levels.map(() => 1)),
519 'marker.size': traceLevel.map(levels => levels.map(() => 7)),
520 }],
521 }];
522
523 for (const hl of levels) {
524 steps.push({
525 label: levelName(hl),
526 method: 'restyle',
527 args: [{
528 'marker.opacity': traceLevel.map(levels => levels.map(l => l === hl ? 1 : 0.12)),
529 'marker.size': traceLevel.map(levels => levels.map(l => l === hl ? 10 : 5)),
530 }],
531 });
532 }
533
534 Plotly.newPlot('chart-pareto', traces, plotLayout({
535 xaxis: { title: 'Compaction Time (s)' },
536 yaxis: { title: 'Size (GiB)' },
537 legend: { title: { text: 'Block Size' } },
538 sliders: [{
539 active: 0,
540 pad: { t: 30 },
541 currentvalue: { prefix: 'zstd level: ', font: { size: 13 } },
542 steps: steps,
543 }],
544 }));
545}
546
547// ── chart 3: dictionary training sweep (with block size slider) ───
548function chartDict() {
549 const traces = [];
550 const traceBlock = [];
551 const trainColors = { 'None': '#45b7d1', '1': '#888', '4': '#4ecdc4', '8': '#26de81',
552 '16': '#f7b731', '32': '#fd9644', '64': '#fc5c65' };
553
554 // find all block sizes that have dict data
555 const blockSizes = [...new Set(
556 data.filter(d => d.dict_bytes > 0 && d.zstd_level === 3 && d.restart_interval === 8)
557 .map(d => d.block_size / 1024)
558 )].sort((a, b) => a - b);
559
560 // collect all dict sizes across all block sizes for consistent x-axis
561 const allDictSizes = [...new Set(
562 data.filter(d => d.dict_bytes > 0 && d.zstd_level === 3 && d.restart_interval === 8)
563 .map(d => d.dict_bytes)
564 )].sort((a, b) => a - b);
565 const xLabels = allDictSizes.map(d => d >= 1024 ? (d / 1024) + 'k' : d + '');
566
567 for (const blockKb of blockSizes) {
568 const bs = blockKb * 1024;
569 const subset = data.filter(d =>
570 d.block_size === bs && d.dict_bytes > 0 &&
571 d.zstd_level === 3 && d.restart_interval === 8
572 );
573 if (subset.length === 0) continue;
574
575 const byTrain = groupBy(subset, d => d.train_mult === null ? 'None' : String(d.train_mult));
576
577 for (const [tm, rows] of Object.entries(byTrain)) {
578 rows.sort((a, b) => a.dict_bytes - b.dict_bytes);
579 const label = tm === 'None' ? 'default' : tm + 'x';
580 traces.push({
581 x: rows.map(d => d.dict_bytes >= 1024 ? (d.dict_bytes / 1024) + 'k' : d.dict_bytes + ''),
582 y: rows.map(gib),
583 name: `${blockKb}k, train=${label}`,
584 mode: 'lines+markers',
585 line: { color: trainColors[tm] || '#888', width: 2.5 },
586 marker: { size: 7 },
587 hovertemplate: `${blockKb}k block, dict=%{x}, train=${label}<br>%{y:.2f} GiB<extra></extra>`,
588 });
589 traceBlock.push(blockKb);
590 }
591
592 // no-dict baseline
593 const baseline = data.find(d =>
594 d.block_size === bs && d.dict_bytes === 0 && d.zstd_level === 3 &&
595 d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10
596 );
597 if (baseline) {
598 traces.push({
599 x: xLabels,
600 y: Array(xLabels.length).fill(gib(baseline)),
601 name: `${blockKb}k, no dict`,
602 mode: 'lines',
603 line: { color: '#ccc', width: 1 },
604 hoverinfo: 'skip',
605 });
606 traceBlock.push(blockKb);
607 }
608 }
609
610 // slider
611 const steps = [{
612 label: 'all',
613 method: 'restyle',
614 args: [{
615 'line.width': traceBlock.map(() => 2.5),
616 'marker.opacity': traceBlock.map(() => 1),
617 'marker.size': traceBlock.map(() => 7),
618 }],
619 }];
620
621 for (const hl of blockSizes) {
622 steps.push({
623 label: `${hl}k`,
624 method: 'restyle',
625 args: [{
626 'line.width': traceBlock.map(b => b === hl ? 3.5 : 1),
627 'marker.opacity': traceBlock.map(b => b === hl ? 1 : 0.15),
628 'marker.size': traceBlock.map(b => b === hl ? 9 : 4),
629 }],
630 });
631 }
632
633 Plotly.newPlot('chart-dict', traces, plotLayout({
634 xaxis: { title: 'Dictionary Size', type: 'category' },
635 yaxis: { title: 'Size (GiB)' },
636 margin: { t: 30, b: 50, l: 60, r: 160 },
637 legend: { x: 1.02, xanchor: 'left', y: 1, font: { size: 10 } },
638 sliders: [{
639 active: 0,
640 pad: { t: 30 },
641 currentvalue: { prefix: 'block size: ', font: { size: 13 } },
642 steps: steps,
643 }],
644 }));
645}
646
647// ── chart 3b: dictionary benefit by block size ────────────────────
648function chartDictBenefit() {
649 // for each block size, find no-dict baseline and best dict config
650 const blocks = [...new Set(data.map(d => d.block_size))].sort((a, b) => a - b);
651 const dictSizes = [16384, 65536, 262144];
652 const colors = { 16384: '#45b7d1', 65536: '#f7b731', 262144: '#fc5c65' };
653
654 const traces = [];
655
656 for (const dictBytes of dictSizes) {
657 const xs = [];
658 const ys = [];
659 const texts = [];
660
661 for (const bs of blocks) {
662 const baseline = data.find(d =>
663 d.block_size === bs && d.dict_bytes === 0 && d.zstd_level === 3 &&
664 d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10
665 );
666 if (!baseline) continue;
667
668 // find best (smallest) result with this dict size at this block size
669 const dictRows = data.filter(d =>
670 d.block_size === bs && d.dict_bytes === dictBytes && d.zstd_level === 3 &&
671 d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10
672 );
673 if (dictRows.length === 0) continue;
674
675 const best = dictRows.reduce((a, b) => a.size_bytes < b.size_bytes ? a : b);
676 const pct = ((baseline.size_bytes - best.size_bytes) / baseline.size_bytes * 100);
677
678 xs.push(bs / 1024 + 'k');
679 ys.push(+pct.toFixed(2));
680 const trainLabel = best.train_mult === null ? 'default' : best.train_mult + 'x';
681 texts.push(`train=${trainLabel}`);
682 }
683
684 if (xs.length > 0) {
685 const dictLabel = dictBytes >= 1024 ? (dictBytes / 1024) + 'k' : dictBytes + '';
686 traces.push({
687 x: xs,
688 y: ys,
689 name: `dict=${dictLabel}`,
690 mode: 'lines+markers',
691 line: { color: colors[dictBytes] || '#888' },
692 marker: { size: 8 },
693 text: texts,
694 hovertemplate: `%{x} block, dict=${dictLabel}<br>%{y:.1f}% smaller<br>%{text}<extra></extra>`,
695 });
696 }
697 }
698
699 Plotly.newPlot('chart-dict-benefit', traces, plotLayout({
700 xaxis: { title: 'Block Size', type: 'category' },
701 yaxis: { title: 'Size Reduction (%)' },
702 }));
703}
704
705// ── chart 3c: dictionary cost (size reduction vs compaction time) ─
706function chartDictCost() {
707 const blockColors = {
708 4096: '#fc5c65', 8192: '#f7b731', 16384: '#45b7d1',
709 32768: '#4ecdc4', 65536: '#a55eea', 131072: '#888',
710 };
711 const blocks = [...new Set(data.map(d => d.block_size))].sort((a, b) => a - b);
712 const traces = [];
713
714 for (const bs of blocks) {
715 const baseline = data.find(d =>
716 d.block_size === bs && d.dict_bytes === 0 && d.zstd_level === 3 &&
717 d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10
718 );
719 if (!baseline) continue;
720
721 const dictRows = data.filter(d =>
722 d.block_size === bs && d.dict_bytes > 0 && d.zstd_level === 3 &&
723 d.restart_interval === 8 && d.opt_filters && d.subcompactions === 10
724 );
725 if (dictRows.length === 0) continue;
726
727 const bsLabel = (bs / 1024) + 'k';
728 traces.push({
729 x: dictRows.map(d => (d.compact_secs - baseline.compact_secs) / baseline.compact_secs * 100),
730 y: dictRows.map(d => (baseline.size_bytes - d.size_bytes) / baseline.size_bytes * 100),
731 name: bsLabel,
732 mode: 'markers',
733 marker: { size: 7, color: blockColors[bs] || '#888' },
734 text: dictRows.map(d => {
735 const dictLabel = d.dict_bytes >= 1024 ? (d.dict_bytes / 1024) + 'k' : d.dict_bytes;
736 const trainLabel = d.train_mult === null ? 'default' : d.train_mult + 'x';
737 return `dict=${dictLabel}, train=${trainLabel}`;
738 }),
739 hovertemplate: `${bsLabel} block<br>%{text}<br>%{y:.1f}% smaller, +%{x:.0f}% time<extra></extra>`,
740 });
741
742 // add the no-dict origin point
743 traces.push({
744 x: [0], y: [0],
745 name: bsLabel + ' (no dict)',
746 mode: 'markers',
747 marker: { size: 9, color: blockColors[bs], symbol: 'x' },
748 showlegend: false,
749 hovertemplate: `${bsLabel} block, no dict<extra></extra>`,
750 });
751 }
752
753 Plotly.newPlot('chart-dict-cost', traces, plotLayout({
754 xaxis: { title: 'Extra Compaction Time (%)' },
755 yaxis: { title: 'Size Reduction (%)' },
756 legend: { title: { text: 'Block Size' } },
757 }));
758}
759
760// ── chart: 32k block deep dive ────────────────────────────────────
761function chart32k() {
762 const bs = 32768;
763 const subset = data.filter(d =>
764 d.block_size === bs && d.zstd_level > 0 && d.restart_interval === 8 &&
765 d.opt_filters && d.subcompactions === 10
766 );
767
768 function dictLabel(d) {
769 if (d.dict_bytes === 0) return 'none';
770 const dk = d.dict_bytes >= 1024 ? (d.dict_bytes / 1024) + 'k' : d.dict_bytes;
771 const tm = d.train_mult === null ? 'def' : d.train_mult + 'x';
772 return `${dk}/${tm}`;
773 }
774
775 // build globally sorted x-axis: none first, then by dict size, then training
776 const allLabels = [...new Set(subset.map(dictLabel))];
777 allLabels.sort((a, b) => {
778 if (a === 'none') return -1;
779 if (b === 'none') return 1;
780 const parseDk = s => parseInt(s) * (s.includes('k/') ? 1024 : 1);
781 const parseTm = s => { const m = s.match(/\/(\d+)x$/); return m ? parseInt(m[1]) : -1; };
782 return parseDk(a) - parseDk(b) || parseTm(a) - parseTm(b);
783 });
784
785 const levels = [...new Set(subset.map(d => d.zstd_level))].sort((a, b) => a - b);
786 const traces = [];
787
788 for (const level of levels) {
789 const rows = subset.filter(d => d.zstd_level === level);
790 rows.sort((a, b) => allLabels.indexOf(dictLabel(a)) - allLabels.indexOf(dictLabel(b)));
791
792 traces.push({
793 x: rows.map(dictLabel),
794 y: rows.map(gib),
795 name: levelName(level),
796 mode: 'lines+markers',
797 line: { color: levelColors[level] || '#888' },
798 marker: { size: 7 },
799 text: rows.map(d => `${d.compact_secs}s`),
800 hovertemplate: `32k, ${levelName(level)}, dict=%{x}<br>%{y:.2f} GiB<br>%{text}<extra></extra>`,
801 });
802 }
803
804 Plotly.newPlot('chart-32k', traces, plotLayout({
805 xaxis: { title: 'Dictionary Config (size/training)', type: 'category',
806 categoryorder: 'array', categoryarray: allLabels },
807 yaxis: { title: 'Size (GiB)' },
808 margin: { t: 30, b: 70, l: 60, r: 20 },
809 }));
810}
811
812// ── chart 4: restart interval ─────────────────────────────────────
813function chartRestart() {
814 const traces = [];
815 const colors = { 4: '#fc5c65', 32: '#45b7d1' };
816
817 for (const blockKb of [4, 32]) {
818 const bs = blockKb * 1024;
819 const subset = data.filter(d =>
820 d.block_size === bs && d.dict_bytes === 0 && d.zstd_level === 3 &&
821 d.opt_filters && d.subcompactions === 10 &&
822 [4, 8, 16, 32, 64].includes(d.restart_interval)
823 ).sort((a, b) => a.restart_interval - b.restart_interval);
824
825 if (subset.length > 0) {
826 traces.push({
827 x: subset.map(d => d.restart_interval),
828 y: subset.map(gib),
829 name: `${blockKb}k blocks`,
830 mode: 'lines+markers',
831 marker: { size: 8 },
832 line: { color: colors[blockKb] },
833 hovertemplate: `${blockKb}k blocks, restart interval %{x}<br>%{y:.2f} GiB<extra></extra>`,
834 });
835 }
836 }
837
838 Plotly.newPlot('chart-restart', traces, plotLayout({
839 xaxis: { title: 'Key Restart Interval', type: 'category' },
840 yaxis: { title: 'Size (GiB)' },
841 legend: { x: 1, xanchor: 'right', y: 1 },
842 }));
843}
844
845// ── chart 5: subcompactions ───────────────────────────────────────
846function chartSubcomp() {
847 const subData = data.filter(d =>
848 d.block_size === 16384 && d.dict_bytes === 0 && d.zstd_level === 3 &&
849 d.restart_interval === 8 && d.opt_filters
850 ).sort((a, b) => a.subcompactions - b.subcompactions);
851
852 if (subData.length > 0) {
853 Plotly.newPlot('chart-subcomp-size', [{
854 x: subData.map(d => d.subcompactions),
855 y: subData.map(gib),
856 mode: 'lines+markers',
857 marker: { size: 8 },
858 line: { color: '#45b7d1' },
859 hovertemplate: 'subcompactions=%{x}<br>%{y:.2f} GiB<extra></extra>',
860 }], plotLayout({
861 xaxis: { title: 'Max Subcompactions', type: 'category' },
862 yaxis: { title: 'Size (GiB)' },
863 showlegend: false,
864 }));
865
866 Plotly.newPlot('chart-subcomp-time', [{
867 x: subData.map(d => d.subcompactions),
868 y: subData.map(d => d.compact_secs),
869 mode: 'lines+markers',
870 marker: { size: 8 },
871 line: { color: '#f7b731' },
872 hovertemplate: 'subcompactions=%{x}<br>%{y:.1f}s<extra></extra>',
873 }], plotLayout({
874 xaxis: { title: 'Max Subcompactions', type: 'category' },
875 yaxis: { title: 'Time (s)' },
876 showlegend: false,
877 }));
878 }
879}
880
881// ── data table ────────────────────────────────────────────────────
882function buildTable(filteredData) {
883 const tbody = document.getElementById('results-body');
884 tbody.innerHTML = filteredData.map(d => `<tr>
885 <td>${d.block_size / 1024}k</td>
886 <td>${levelName(d.zstd_level)}</td>
887 <td>${d.restart_interval}</td>
888 <td>${d.dict_bytes}</td>
889 <td>${d.train_mult === null ? '' : d.train_mult + 'x'}</td>
890 <td>${d.opt_filters ? 'on' : 'off'}</td>
891 <td>${d.subcompactions}</td>
892 <td><strong>${(d.size_mib / 1024).toFixed(2)}</strong></td>
893 <td>${d.compact_secs.toFixed(1)}</td>
894 </tr>`).join('');
895}
896
897function setupFilters() {
898 const blocks = [...new Set(data.map(d => d.block_size))].sort((a, b) => a - b);
899 const levels = [...new Set(data.map(d => d.zstd_level))].sort((a, b) => a - b);
900 const dicts = [...new Set(data.map(d => d.dict_bytes))].sort((a, b) => a - b);
901
902 const fBlock = document.getElementById('f-block');
903 const fLevel = document.getElementById('f-level');
904 const fDict = document.getElementById('f-dict');
905
906 blocks.forEach(b => { const o = document.createElement('option'); o.value = b; o.textContent = (b/1024) + 'k'; fBlock.appendChild(o); });
907 levels.forEach(l => { const o = document.createElement('option'); o.value = l; o.textContent = levelName(l); fLevel.appendChild(o); });
908 dicts.forEach(d => { const o = document.createElement('option'); o.value = d; o.textContent = d; fDict.appendChild(o); });
909
910 const applyFilters = () => {
911 let filtered = data;
912 if (fBlock.value) filtered = filtered.filter(d => d.block_size === +fBlock.value);
913 if (fLevel.value) filtered = filtered.filter(d => d.zstd_level === +fLevel.value);
914 if (fDict.value !== '') filtered = filtered.filter(d => d.dict_bytes === +fDict.value);
915 buildTable(filtered);
916 };
917
918 [fBlock, fLevel, fDict].forEach(el => el.addEventListener('change', applyFilters));
919}
920
921// sortable headers
922let sortCol = 'size_mib', sortAsc = true;
923document.querySelectorAll('#results-table th').forEach(th => {
924 th.addEventListener('click', () => {
925 const col = th.dataset.col;
926 if (sortCol === col) sortAsc = !sortAsc;
927 else { sortCol = col; sortAsc = true; }
928 const filtered = getFilteredData();
929 filtered.sort((a, b) => {
930 const av = a[col] ?? -1, bv = b[col] ?? -1;
931 return sortAsc ? av - bv : bv - av;
932 });
933 buildTable(filtered);
934 });
935});
936
937function getFilteredData() {
938 let filtered = [...data];
939 const fBlock = document.getElementById('f-block').value;
940 const fLevel = document.getElementById('f-level').value;
941 const fDict = document.getElementById('f-dict').value;
942 if (fBlock) filtered = filtered.filter(d => d.block_size === +fBlock);
943 if (fLevel) filtered = filtered.filter(d => d.zstd_level === +fLevel);
944 if (fDict !== '') filtered = filtered.filter(d => d.dict_bytes === +fDict);
945 return filtered;
946}
947
948// ── CSV download ──────────────────────────────────────────────────
949document.getElementById('download-csv').addEventListener('click', () => {
950 const blob = new Blob([csvText], { type: 'text/csv' });
951 const a = document.createElement('a');
952 a.href = URL.createObjectURL(blob);
953 a.download = 'sweep-results.csv';
954 a.click();
955 URL.revokeObjectURL(a.href);
956});
957
958// ── zero-Y toggles ────────────────────────────────────────────────
959document.querySelectorAll('.zero-toggle input').forEach(cb => {
960 cb.checked = false; // override browser autofill on refresh
961 cb.addEventListener('change', () => {
962 const chartId = cb.dataset.chart;
963 const mode = cb.checked ? 'tozero' : 'normal';
964 Plotly.relayout(chartId, { 'yaxis.rangemode': mode });
965 });
966});
967
968// ── data profile histograms ────────────────────────────────────────
969function fmtBytes(n) {
970 if (n >= 1048576) return (n / 1048576).toFixed(0) + 'M';
971 if (n >= 1024) return (n / 1024).toFixed(0) + 'k';
972 return n + '';
973}
974
975function chartValueHist() {
976 const h = stats.value_size_hist;
977 Plotly.newPlot('chart-value-hist', [{
978 x: h.map(([lo, hi]) => `${fmtBytes(lo)}-${fmtBytes(hi)}`),
979 y: h.map(([,, c]) => c),
980 type: 'bar',
981 marker: { color: '#45b7d1' },
982 hovertemplate: '%{x} bytes<br>%{y:,} keys<extra></extra>',
983 }], plotLayout({
984 xaxis: { title: 'Value Size (bytes)', type: 'category' },
985 yaxis: { title: 'Keys', type: 'log' },
986 showlegend: false,
987 }));
988}
989
990function chartAccountHist() {
991 const h = stats.keys_per_account_hist;
992 Plotly.newPlot('chart-account-hist', [{
993 x: h.map(([lo, hi]) => `${fmtNum(lo)}-${fmtNum(hi)}`),
994 y: h.map(([,, c]) => c),
995 type: 'bar',
996 marker: { color: '#4ecdc4' },
997 hovertemplate: '%{x} keys<br>%{y:,} accounts<extra></extra>',
998 }], plotLayout({
999 xaxis: { title: 'Keys per Account', type: 'category' },
1000 yaxis: { title: 'Accounts', type: 'log' },
1001 showlegend: false,
1002 }));
1003}
1004
1005// ── init ──────────────────────────────────────────────────────────
1006chartValueHist();
1007chartAccountHist();
1008chartBlockSize();
1009chartPareto();
1010chartDict();
1011chartDictBenefit();
1012chartDictCost();
1013chart32k();
1014chartRestart();
1015chartSubcomp();
1016setupFilters();
1017buildTable(data);
1018</script>
1019</body>
1020</html>