rerun with a 1%-ish network sample · microcosm.blue/hubble@2b78afd

+175 -151

space-efficiency-check/index.html

··· 106 106 <script> 107 107 // ── embed CSV data ──────────────────────────────────────────────── 108 108 const csvText = `block_size,zstd_level,restart_interval,dict_bytes,train_mult,opt_filters,subcompactions,size_bytes,size_mib,compact_secs 109 - 4096,1,8,0,,true,10,11079791854,10566,16.2 110 - 4096,1,8,16384,16,true,10,10365069306,9884,20.8 111 - 4096,3,4,0,,true,10,11208454885,10689,24.0 112 - 4096,3,8,0,,true,10,11070990214,10558,20.2 113 - 4096,3,8,1024,,true,10,10617592768,10125,27.7 114 - 4096,3,8,1024,4,true,10,11071036189,10558,20.5 115 - 4096,3,8,1024,16,true,10,11071022120,10558,21.0 116 - 4096,3,8,1024,64,true,10,10782814873,10283,27.3 117 - 4096,3,8,4096,,true,10,10598956252,10107,30.5 118 - 4096,3,8,4096,4,true,10,11071016375,10558,20.8 119 - 4096,3,8,4096,16,true,10,10271883281,9796,25.6 120 - 4096,3,8,4096,64,true,10,10213224021,9740,29.1 121 - 4096,3,8,16384,,true,10,10556340899,10067,30.9 122 - 4096,3,8,16384,4,true,10,10301070056,9823,27.1 123 - 4096,3,8,16384,16,true,10,10227379189,9753,30.4 124 - 4096,3,8,16384,64,true,10,10213306710,9740,33.9 125 - 4096,3,8,65536,,true,10,10465976570,9981,33.0 126 - 4096,3,8,65536,4,true,10,10217242670,9743,30.0 127 - 4096,3,8,65536,16,true,10,10164362837,9693,34.1 128 - 4096,3,8,65536,64,true,10,10139725001,9669,38.2 129 - 4096,3,16,0,,true,10,10974836115,10466,25.6 130 - 4096,3,32,0,,true,10,10931177138,10424,21.3 131 - 4096,3,64,0,,true,10,10931160646,10424,20.8 132 - 4096,6,8,0,,true,10,10895448640,10390,33.9 133 - 4096,6,8,16384,16,true,10,10093706553,9626,43.9 134 - 4096,9,8,0,,true,10,10893183784,10388,58.0 135 - 4096,9,8,16384,16,true,10,9946861812,9486,58.1 136 - 8192,1,8,0,,true,10,10028074705,9563,20.8 137 - 8192,1,8,16384,16,true,10,9811325586,9356,17.4 138 - 8192,3,8,0,,false,10,10040289523,9575,18.0 139 - 8192,3,8,0,,true,10,10040284381,9575,16.2 140 - 8192,3,8,1024,,true,10,9835665629,9380,25.4 141 - 8192,3,8,1024,4,true,10,10040279221,9575,16.8 142 - 8192,3,8,1024,16,true,10,10040292654,9575,16.8 143 - 8192,3,8,1024,64,true,10,10119180082,9650,25.0 144 - 8192,3,8,4096,,true,10,9848434052,9392,24.6 145 - 8192,3,8,4096,4,true,10,10040294086,9575,16.8 146 - 8192,3,8,4096,16,true,10,9717871364,9267,22.5 147 - 8192,3,8,4096,64,true,10,9665939593,9218,25.4 148 - 8192,3,8,16384,,true,10,9859705853,9402,25.6 149 - 8192,3,8,16384,4,true,10,9782101780,9328,24.5 150 - 8192,3,8,16384,16,true,10,9713733091,9263,26.0 151 - 8192,3,8,16384,64,true,10,9694061307,9244,27.5 152 - 8192,3,8,65536,,true,10,9849121697,9392,26.2 153 - 8192,3,8,65536,4,true,10,9745808012,9294,24.9 154 - 8192,3,8,65536,16,true,10,9696039387,9246,27.5 155 - 8192,3,8,65536,64,true,10,9672733565,9224,31.5 156 - 8192,6,8,0,,true,10,9874369448,9416,31.7 157 - 8192,6,8,16384,16,true,10,9555000218,9112,38.5 158 - 16384,1,8,0,,true,10,9401904407,8966,15.6 159 - 16384,1,8,16384,16,true,10,9384259110,8949,15.2 160 - 16384,3,8,0,,true,4,9439746342,9002,22.6 161 - 16384,3,8,0,,true,10,9439798265,9002,14.1 162 - 16384,3,8,16384,16,true,10,9336363403,8903,21.6 163 - 16384,6,8,0,,true,10,9238560723,8810,28.8 164 - 16384,6,8,16384,16,true,10,9157064709,8732,35.4 165 - 16384,9,8,0,,true,10,9227102912,8799,55.0 166 - 16384,9,8,16384,16,true,10,9002507344,8585,48.0 167 - 32768,1,8,0,,true,10,9161174067,8736,12.6 168 - 32768,1,8,16384,16,true,10,9129717484,8706,13.8 169 - 32768,3,4,0,,true,10,9186445992,8760,13.7 170 - 32768,3,8,0,,false,10,9074744264,8654,13.3 171 - 32768,3,8,0,,true,10,9074587560,8654,13.3 172 - 32768,3,8,16384,16,true,10,9060352672,8640,22.0 173 - 32768,3,16,0,,true,10,9009532082,8592,13.8 174 - 32768,3,32,0,,true,10,8974169294,8558,13.3 175 - 32768,3,64,0,,true,10,8953489124,8538,13.3 176 - 32768,6,8,0,,true,10,8959226119,8544,27.5 177 - 32768,6,8,16384,16,true,10,8891715689,8479,34.9 178 - 65536,1,8,0,,true,10,8940392508,8526,11.5 179 - 65536,1,8,16384,16,true,10,8940404881,8526,10.8 180 - 65536,3,8,0,,true,10,8861096723,8450,12.9 181 - 65536,3,8,16384,16,true,10,8861073512,8450,13.4 182 - 65536,6,8,0,,true,10,8742392375,8337,27.9 183 - 65536,6,8,16384,16,true,10,8742335065,8337,29.1 184 - 65536,9,8,0,,true,10,8633639539,8233,36.6 185 - 65536,9,8,16384,16,true,10,8633630034,8233,37.0 186 - 8192,9,8,0,,true,10,9869739461,9412,50.2 187 - 8192,9,8,16384,16,true,10,9399964893,8964,54.3 188 - 32768,9,8,0,,true,10,8857646963,8447,42.4 189 - 32768,9,8,16384,16,true,10,8747823404,8342,51.4 190 - 16384,3,8,0,,true,1,9439775263,9002,81.6 191 - 4096,0,8,0,,true,10,15722034231,14993,13.5 192 - 8192,0,8,0,,true,10,14715967944,14034,12.8 193 - 16384,0,8,0,,true,10,14045987598,13395,11.7 194 - 32768,0,8,0,,true,10,13597519906,12967,12.9 195 - 65536,0,8,0,,true,10,13332609861,12714,16.7 196 - 4096,3,8,1024,8,true,10,11071194210,10558,16.7 197 - 4096,3,8,1024,32,true,10,10803699447,10303,27.1 198 - 4096,3,8,2048,,true,10,10618567648,10126,30.1 199 - 4096,3,8,2048,4,true,10,11070980059,10558,21.3 200 - 4096,3,8,2048,8,true,10,11070998773,10558,21.2 201 - 4096,3,8,2048,16,true,10,10344211112,9865,25.7 202 - 4096,3,8,2048,32,true,10,10310192553,9832,29.4 203 - 4096,3,8,2048,64,true,10,10275154282,9799,29.1 204 - 4096,3,8,4096,8,true,10,10323537489,9845,30.1 205 - 4096,3,8,4096,32,true,10,10235055391,9760,29.2 206 - 4096,3,8,8192,,true,10,10554955403,10065,30.6 207 - 4096,3,8,8192,4,true,10,10343018691,9863,26.8 208 - 4096,3,8,8192,8,true,10,10259861034,9784,29.0 209 - 4096,3,8,8192,16,true,10,10226087656,9752,29.2 210 - 4096,3,8,8192,32,true,10,10208952277,9736,29.4 211 - 4096,3,8,8192,64,true,10,10201640423,9729,29.8 212 - 4096,3,8,16384,8,true,10,10249137059,9774,30.1 213 - 4096,3,8,16384,32,true,10,10224396141,9750,30.7 214 - 4096,3,8,65536,8,true,10,10187080812,9715,30.9 215 - 4096,3,8,65536,32,true,10,10154462199,9684,33.1 216 - 8192,3,8,1024,8,true,10,10040350281,9575,22.2 217 - 8192,3,8,1024,32,true,10,10040330045,9575,16.4 218 - 8192,3,8,2048,,true,10,9829590607,9374,23.4 219 - 8192,3,8,2048,4,true,10,10040326898,9575,16.5 220 - 8192,3,8,2048,8,true,10,10040331586,9575,16.5 221 - 8192,3,8,2048,16,true,10,10040326454,9575,16.5 222 - 8192,3,8,2048,32,true,10,10107002899,9638,23.1 223 - 8192,3,8,2048,64,true,10,10089763846,9622,24.7 224 - 8192,3,8,4096,8,true,10,10040335592,9575,18.1 225 - 8192,3,8,4096,32,true,10,9694641840,9245,22.3 226 - 8192,3,8,8192,,true,10,9844667804,9388,24.5 227 - 8192,3,8,8192,4,true,10,10040337106,9575,16.8 228 - 8192,3,8,8192,8,true,10,9746958438,9295,23.3 229 - 8192,3,8,8192,16,true,10,9708134370,9258,25.7 230 - 8192,3,8,8192,32,true,10,9682607740,9234,25.7 231 - 8192,3,8,8192,64,true,10,9674036954,9225,26.2 232 - 8192,3,8,16384,8,true,10,9737625952,9286,26.8 233 - 8192,3,8,16384,32,true,10,9699236736,9249,26.5 234 - 8192,3,8,65536,8,true,10,9710583278,9260,26.5 235 - 8192,3,8,65536,32,true,10,9683941181,9235,28.6 236 - 131072,0,8,0,,true,10,13201586596,12590,13.5 237 - 131072,1,8,0,,true,10,8823092338,8414,10.4 238 - 131072,1,8,16384,16,true,10,8823145617,8414,10.1 239 - 131072,3,8,0,,true,10,8712125430,8308,12.5 240 - 131072,3,8,16384,16,true,10,8712317883,8308,12.9 241 - 131072,6,8,0,,true,10,8600177778,8201,27.0 242 - 131072,6,8,16384,16,true,10,8600107633,8201,27.6 243 - 131072,9,8,0,,true,10,8487562590,8094,38.2 244 - 131072,9,8,16384,16,true,10,8487490042,8094,38.8 109 + 4096,0,8,0,,true,10,31395311799,29940,19.8 110 + 4096,1,8,0,,true,10,22116416091,21091,30.7 111 + 4096,1,8,16384,16,true,10,20659329178,19702,42.5 112 + 4096,3,4,0,,true,10,22374836755,21338,51.4 113 + 4096,3,8,0,,true,10,22100643559,21076,41.7 114 + 4096,3,8,1024,,true,10,21177970088,20196,57.0 115 + 4096,3,8,1024,4,true,10,22100668085,21076,42.2 116 + 4096,3,8,1024,8,true,10,22100676471,21076,43.2 117 + 4096,3,8,1024,16,true,10,22100669290,21076,42.9 118 + 4096,3,8,1024,32,true,10,21526349249,20529,55.1 119 + 4096,3,8,1024,64,true,10,21501054074,20505,60.0 120 + 4096,3,8,2048,,true,10,21184609495,20203,59.8 121 + 4096,3,8,2048,4,true,10,22100673247,21076,42.1 122 + 4096,3,8,2048,8,true,10,22100621663,21076,41.2 123 + 4096,3,8,2048,16,true,10,20635793718,19679,52.0 124 + 4096,3,8,2048,32,true,10,20562444599,19609,58.2 125 + 4096,3,8,2048,64,true,10,20508962586,19558,57.6 126 + 4096,3,8,4096,,true,10,21128768715,20149,61.0 127 + 4096,3,8,4096,4,true,10,22100685331,21076,41.4 128 + 4096,3,8,4096,8,true,10,20577552594,19624,51.9 129 + 4096,3,8,4096,16,true,10,20479709404,19530,60.2 130 + 4096,3,8,4096,32,true,10,20403721320,19458,60.9 131 + 4096,3,8,4096,64,true,10,20368652511,19425,62.7 132 + 4096,3,8,8192,,true,10,21057872538,20082,62.9 133 + 4096,3,8,8192,4,true,10,20632375521,19676,54.1 134 + 4096,3,8,8192,8,true,10,20506735960,19556,61.2 135 + 4096,3,8,8192,16,true,10,20390435178,19445,62.1 136 + 4096,3,8,8192,32,true,10,20363765340,19420,61.9 137 + 4096,3,8,8192,64,true,10,20331974936,19390,61.6 138 + 4096,3,8,16384,,true,10,21083190741,20106,65.0 139 + 4096,3,8,16384,4,true,10,20579253867,19625,54.2 140 + 4096,3,8,16384,8,true,10,20455471802,19507,61.6 141 + 4096,3,8,16384,16,true,10,20405638509,19460,63.1 142 + 4096,3,8,16384,32,true,10,20386866978,19442,65.3 143 + 4096,3,8,16384,64,true,10,20367953764,19424,67.5 144 + 4096,3,8,65536,,true,10,20893557686,19925,63.9 145 + 4096,3,8,65536,4,true,10,20409555824,19464,56.4 146 + 4096,3,8,65536,8,true,10,20345337995,19402,64.1 147 + 4096,3,8,65536,16,true,10,20298445731,19358,65.1 148 + 4096,3,8,65536,32,true,10,20261884271,19323,69.0 149 + 4096,3,8,65536,64,true,10,20239324109,19301,74.3 150 + 4096,3,16,0,,true,10,21909626509,20894,48.0 151 + 4096,3,32,0,,true,10,21823083686,20812,39.5 152 + 4096,3,64,0,,true,10,21823091636,20812,40.4 153 + 4096,6,8,0,,true,10,21750720790,20743,65.2 154 + 4096,6,8,16384,16,true,10,20146634422,19213,84.1 155 + 4096,9,8,0,,true,10,21746294515,20738,111.8 156 + 4096,9,8,16384,16,true,10,19849302063,18929,116.6 157 + 8192,0,8,0,,true,10,29379196929,28018,35.0 158 + 8192,1,8,0,,true,10,20005967746,19079,25.5 159 + 8192,1,8,16384,16,true,10,19572338449,18665,34.0 160 + 8192,3,8,0,,false,10,20031898121,19103,36.1 161 + 8192,3,8,0,,true,10,20031920603,19103,31.9 162 + 8192,3,8,1024,,true,10,19622953913,18713,48.9 163 + 8192,3,8,1024,4,true,10,20031917101,19103,33.8 164 + 8192,3,8,1024,8,true,10,20031918966,19103,33.5 165 + 8192,3,8,1024,16,true,10,20031904492,19103,33.5 166 + 8192,3,8,1024,32,true,10,20031915783,19103,33.1 167 + 8192,3,8,1024,64,true,10,20182475006,19247,50.1 168 + 8192,3,8,2048,,true,10,19620130060,18711,52.0 169 + 8192,3,8,2048,4,true,10,20031954949,19103,35.7 170 + 8192,3,8,2048,8,true,10,20031939452,19103,33.7 171 + 8192,3,8,2048,16,true,10,20031949661,19103,33.4 172 + 8192,3,8,2048,32,true,10,20143626622,19210,47.8 173 + 8192,3,8,2048,64,true,10,20120486239,19188,52.3 174 + 8192,3,8,4096,,true,10,19635342683,18725,48.9 175 + 8192,3,8,4096,4,true,10,20031954812,19103,33.2 176 + 8192,3,8,4096,8,true,10,20031942177,19103,33.3 177 + 8192,3,8,4096,16,true,10,19372492949,18475,47.6 178 + 8192,3,8,4096,32,true,10,19313920339,18419,50.4 179 + 8192,3,8,4096,64,true,10,19282505585,18389,52.1 180 + 8192,3,8,8192,,true,10,19635635166,18726,50.9 181 + 8192,3,8,8192,4,true,10,20031928773,19103,35.0 182 + 8192,3,8,8192,8,true,10,19410703858,18511,46.3 183 + 8192,3,8,8192,16,true,10,19341160203,18445,50.9 184 + 8192,3,8,8192,32,true,10,19296132462,18402,50.9 185 + 8192,3,8,8192,64,true,10,19273631354,18380,51.8 186 + 8192,3,8,16384,,true,10,19655701449,18745,54.3 187 + 8192,3,8,16384,4,true,10,19502525484,18599,50.5 188 + 8192,3,8,16384,8,true,10,19419249737,18519,51.3 189 + 8192,3,8,16384,16,true,10,19358437498,18461,52.4 190 + 8192,3,8,16384,32,true,10,19340825956,18444,53.1 191 + 8192,3,8,16384,64,true,10,19329911907,18434,54.7 192 + 8192,3,8,65536,,true,10,19654121838,18743,52.6 193 + 8192,3,8,65536,4,true,10,19436643569,18536,50.1 194 + 8192,3,8,65536,8,true,10,19376580371,18478,54.5 195 + 8192,3,8,65536,16,true,10,19337019622,18441,55.6 196 + 8192,3,8,65536,32,true,10,19318086405,18423,59.0 197 + 8192,3,8,65536,64,true,10,19305582440,18411,64.8 198 + 8192,6,8,0,,true,10,19702719743,18789,62.2 199 + 8192,6,8,16384,16,true,10,19065986792,18182,74.2 200 + 8192,9,8,0,,true,10,19693493308,18781,100.9 201 + 8192,9,8,16384,16,true,10,18747260904,17878,100.7 202 + 16384,0,8,0,,true,10,28042902316,26743,26.0 203 + 16384,1,8,0,,true,10,18751959286,17883,22.9 204 + 16384,1,8,16384,16,true,10,18714866778,17847,30.2 205 + 16384,3,8,0,,true,1,18829928785,17957,174.2 206 + 16384,3,8,0,,true,4,18829973330,17957,44.1 207 + 16384,3,8,0,,true,10,18830038815,17957,27.8 208 + 16384,3,8,16384,16,true,10,18615498021,17753,43.5 209 + 16384,6,8,0,,true,10,18431835527,17577,57.9 210 + 16384,6,8,16384,16,true,10,18267813872,17421,69.3 211 + 16384,9,8,0,,true,10,18408979927,17556,109.1 212 + 16384,9,8,16384,16,true,10,17956014400,17124,95.6 213 + 32768,0,8,0,,true,10,27149041457,25891,22.3 214 + 32768,1,8,0,,true,10,18270567659,17424,21.6 215 + 32768,1,8,16384,16,true,10,18211125411,17367,27.5 216 + 32768,3,4,0,,true,10,18324784539,17475,28.5 217 + 32768,3,8,0,,false,10,18101433642,17262,27.5 218 + 32768,3,8,0,,true,10,18101394905,17262,26.5 219 + 32768,3,8,16384,16,true,10,18076847819,17239,46.5 220 + 32768,3,16,0,,true,10,17971384889,17138,28.1 221 + 32768,3,32,0,,true,10,17900644184,17071,27.1 222 + 32768,3,64,0,,true,10,17859565644,17032,29.2 223 + 32768,6,8,0,,true,10,17875945091,17047,56.9 224 + 32768,6,8,16384,16,true,10,17743235923,16921,72.5 225 + 32768,9,8,0,,true,10,17672322648,16853,76.0 226 + 32768,9,8,16384,16,true,10,17455398843,16646,96.0 227 + 65536,0,8,0,,true,10,26622152522,25388,21.2 228 + 65536,1,8,0,,true,10,17829479045,17003,21.0 229 + 65536,1,8,16384,16,true,10,17829619661,17003,21.4 230 + 65536,3,8,0,,true,10,17676395890,16857,25.5 231 + 65536,3,8,16384,16,true,10,17676353730,16857,26.4 232 + 65536,6,8,0,,true,10,17445895583,16637,53.6 233 + 65536,6,8,16384,16,true,10,17445851139,16637,54.6 234 + 65536,9,8,0,,true,10,17228573735,16430,73.1 235 + 65536,9,8,16384,16,true,10,17228513145,16430,73.9 236 + 131072,0,8,0,,true,10,26362266962,25141,19.5 237 + 131072,1,8,0,,true,10,17595617600,16780,20.4 238 + 131072,1,8,16384,16,true,10,17595914003,16780,20.0 239 + 131072,3,8,0,,true,10,17381327820,16576,24.9 240 + 131072,3,8,16384,16,true,10,17381302914,16576,25.5 241 + 131072,6,8,0,,true,10,17164905178,16369,54.9 242 + 131072,6,8,16384,16,true,10,17165045658,16369,56.9 243 + 131072,9,8,0,,true,10,16940090482,16155,76.5 244 + 131072,9,8,16384,16,true,10,16940006516,16155,76.7 245 245 `; 246 246 247 247 // ── parse ───────────────────────────────────────────────────────── ··· 269 269 270 270 // ── data profile stats ─────────────────────────────────────────── 271 271 const stats = { 272 - total_keys: 110471379, 273 - total_accounts: 78568, 274 - value_size_hist: [ 275 - [16, 32, 32], [32, 64, 30165], [64, 128, 13339156], [128, 256, 79513656], 276 - [256, 512, 13262139], [512, 1024, 3916643], [1024, 2048, 384505], 277 - [2048, 4096, 20803], [4096, 8192, 3652], [8192, 16384, 472], 278 - [16384, 32768, 68], [32768, 65536, 44], [65536, 131072, 37], [131072, 262144, 7], 279 - ], 280 - keys_per_account_hist: [ 281 - [1, 2, 2630], [2, 4, 8451], [4, 8, 6955], [8, 16, 7082], [16, 32, 7424], 282 - [32, 64, 7898], [64, 128, 7817], [128, 256, 7196], [256, 512, 6323], 283 - [512, 1024, 5230], [1024, 2048, 4087], [2048, 4096, 2869], [4096, 8192, 2038], 284 - [8192, 16384, 1223], [16384, 32768, 747], [32768, 65536, 379], 285 - [65536, 131072, 165], [131072, 262144, 45], [262144, 524288, 6], 286 - [524288, 1048576, 2], [1048576, 2097152, 1], 272 + "total_keys": 220118913, 273 + "total_accounts": 155536, 274 + "value_size_hist": [ 275 + [16, 32, 62], 276 + [32, 64, 59915], 277 + [64, 128, 24326635], 278 + [128, 256, 160003952], 279 + [256, 512, 27257670], 280 + [512, 1024, 7251053], 281 + [1024, 2048, 1189568], 282 + [2048, 4096, 28324], 283 + [4096, 8192, 1495], 284 + [8192, 16384, 188], 285 + [16384, 32768, 37], 286 + [32768, 65536, 11], 287 + [65536, 131072, 2], 288 + [131072, 262144, 1] 287 289 ], 290 + "keys_per_account_hist": [ 291 + [1, 2, 5192], 292 + [2, 4, 16553], 293 + [4, 8, 13913], 294 + [8, 16, 13736], 295 + [16, 32, 15006], 296 + [32, 64, 15720], 297 + [64, 128, 15330], 298 + [128, 256, 14137], 299 + [256, 512, 12703], 300 + [512, 1024, 10464], 301 + [1024, 2048, 8027], 302 + [2048, 4096, 5631], 303 + [4096, 8192, 3899], 304 + [8192, 16384, 2536], 305 + [16384, 32768, 1463], 306 + [32768, 65536, 789], 307 + [65536, 131072, 322], 308 + [131072, 262144, 92], 309 + [262144, 524288, 20], 310 + [524288, 1048576, 3] 311 + ] 288 312 }; 289 313 290 314 const fmtNum = n => n.toLocaleString();

+46

space-efficiency-check/readme.md

··· 2 2 3 3 assessing rocksdb's space efficiency 4 4 5 + 1. grab a lot of car files (i downloaded all of morel) 6 + 7 + 2. load them into rocks, i did 33% sampling for ~1% total network or something. (probably even better to sample when grabbing cars) 8 + 9 + ```bash 10 + RUST_LOG=info cargo run --release -- \ 11 + --db-dir /path/to/new/rocks/db \ 12 + --sample 0.33 \ 13 + /path/to/cars 14 + ``` 15 + 16 + 3. we can skip running `sample.rs` since we sampled at step 1 17 + 18 + 4. get stats (may as well do this before sweep since it doesn't take too long) 19 + 20 + ```bash 21 + cargo run --release --example stats -- \ 22 + /path/to/new/rocks/db > stats.json 23 + ``` 24 + 25 + and then copy them over to the html file. 26 + 27 + 5. do the sweeps (takes forever) 28 + 29 + ```bash 30 + cargo run --release --example sweep -- \ 31 + /path/to/new/rocks/db --output sweep-results.csv 32 + ``` 33 + 34 + sweep will read the output file to avoid rerunning the same config. delete a line if you actually do want to rereun a config. 35 + 36 + copy the csv into the html. 37 + 38 + #### notes 39 + 40 + - take all timing measurements with a heap of salt: this is not rigourous benchmarking, just first-order vibes 41 + - only one timing measurement is taken for each config 42 + - i'm not running on any kind of controlled environment (doing other things while it goes) 43 + - my disk doesn't have a lot of free space which probabaly messes with it unpredictably 44 + - each run has to work with the previous run's config when recompacting! most of the work is on the write side, but the runs aren't fully independent. 45 + 46 + 47 + -------- 48 + 49 + old stuff: 50 + 5 51 6 52 ### first full run: zstd for layers 2+ 7 53

+36 -37

space-efficiency-check/stats.json

··· 1 1 { 2 - "total_keys": 110471379, 3 - "total_accounts": 78568, 2 + "total_keys": 220118913, 3 + "total_accounts": 155536, 4 4 "value_size_hist": [ 5 - [16, 32, 32], 6 - [32, 64, 30165], 7 - [64, 128, 13339156], 8 - [128, 256, 79513656], 9 - [256, 512, 13262139], 10 - [512, 1024, 3916643], 11 - [1024, 2048, 384505], 12 - [2048, 4096, 20803], 13 - [4096, 8192, 3652], 14 - [8192, 16384, 472], 15 - [16384, 32768, 68], 16 - [32768, 65536, 44], 17 - [65536, 131072, 37], 18 - [131072, 262144, 7] 5 + [16, 32, 62], 6 + [32, 64, 59915], 7 + [64, 128, 24326635], 8 + [128, 256, 160003952], 9 + [256, 512, 27257670], 10 + [512, 1024, 7251053], 11 + [1024, 2048, 1189568], 12 + [2048, 4096, 28324], 13 + [4096, 8192, 1495], 14 + [8192, 16384, 188], 15 + [16384, 32768, 37], 16 + [32768, 65536, 11], 17 + [65536, 131072, 2], 18 + [131072, 262144, 1] 19 19 ], 20 20 "keys_per_account_hist": [ 21 - [1, 2, 2630], 22 - [2, 4, 8451], 23 - [4, 8, 6955], 24 - [8, 16, 7082], 25 - [16, 32, 7424], 26 - [32, 64, 7898], 27 - [64, 128, 7817], 28 - [128, 256, 7196], 29 - [256, 512, 6323], 30 - [512, 1024, 5230], 31 - [1024, 2048, 4087], 32 - [2048, 4096, 2869], 33 - [4096, 8192, 2038], 34 - [8192, 16384, 1223], 35 - [16384, 32768, 747], 36 - [32768, 65536, 379], 37 - [65536, 131072, 165], 38 - [131072, 262144, 45], 39 - [262144, 524288, 6], 40 - [524288, 1048576, 2], 41 - [1048576, 2097152, 1] 21 + [1, 2, 5192], 22 + [2, 4, 16553], 23 + [4, 8, 13913], 24 + [8, 16, 13736], 25 + [16, 32, 15006], 26 + [32, 64, 15720], 27 + [64, 128, 15330], 28 + [128, 256, 14137], 29 + [256, 512, 12703], 30 + [512, 1024, 10464], 31 + [1024, 2048, 8027], 32 + [2048, 4096, 5631], 33 + [4096, 8192, 3899], 34 + [8192, 16384, 2536], 35 + [16384, 32768, 1463], 36 + [32768, 65536, 789], 37 + [65536, 131072, 322], 38 + [131072, 262144, 92], 39 + [262144, 524288, 20], 40 + [524288, 1048576, 3] 42 41 ] 43 42 }

+136 -136

space-efficiency-check/sweep-results.csv

··· 1 1 block_size,zstd_level,restart_interval,dict_bytes,train_mult,opt_filters,subcompactions,size_bytes,size_mib,compact_secs 2 - 4096,1,8,0,,true,10,11079791854,10566,16.2 3 - 4096,1,8,16384,16,true,10,10365069306,9884,20.8 4 - 4096,3,4,0,,true,10,11208454885,10689,24.0 5 - 4096,3,8,0,,true,10,11070990214,10558,20.2 6 - 4096,3,8,1024,,true,10,10617592768,10125,27.7 7 - 4096,3,8,1024,4,true,10,11071036189,10558,20.5 8 - 4096,3,8,1024,16,true,10,11071022120,10558,21.0 9 - 4096,3,8,1024,64,true,10,10782814873,10283,27.3 10 - 4096,3,8,4096,,true,10,10598956252,10107,30.5 11 - 4096,3,8,4096,4,true,10,11071016375,10558,20.8 12 - 4096,3,8,4096,16,true,10,10271883281,9796,25.6 13 - 4096,3,8,4096,64,true,10,10213224021,9740,29.1 14 - 4096,3,8,16384,,true,10,10556340899,10067,30.9 15 - 4096,3,8,16384,4,true,10,10301070056,9823,27.1 16 - 4096,3,8,16384,16,true,10,10227379189,9753,30.4 17 - 4096,3,8,16384,64,true,10,10213306710,9740,33.9 18 - 4096,3,8,65536,,true,10,10465976570,9981,33.0 19 - 4096,3,8,65536,4,true,10,10217242670,9743,30.0 20 - 4096,3,8,65536,16,true,10,10164362837,9693,34.1 21 - 4096,3,8,65536,64,true,10,10139725001,9669,38.2 22 - 4096,3,16,0,,true,10,10974836115,10466,25.6 23 - 4096,3,32,0,,true,10,10931177138,10424,21.3 24 - 4096,3,64,0,,true,10,10931160646,10424,20.8 25 - 4096,6,8,0,,true,10,10895448640,10390,33.9 26 - 4096,6,8,16384,16,true,10,10093706553,9626,43.9 27 - 4096,9,8,0,,true,10,10893183784,10388,58.0 28 - 4096,9,8,16384,16,true,10,9946861812,9486,58.1 29 - 8192,1,8,0,,true,10,10028074705,9563,20.8 30 - 8192,1,8,16384,16,true,10,9811325586,9356,17.4 31 - 8192,3,8,0,,false,10,10040289523,9575,18.0 32 - 8192,3,8,0,,true,10,10040284381,9575,16.2 33 - 8192,3,8,1024,,true,10,9835665629,9380,25.4 34 - 8192,3,8,1024,4,true,10,10040279221,9575,16.8 35 - 8192,3,8,1024,16,true,10,10040292654,9575,16.8 36 - 8192,3,8,1024,64,true,10,10119180082,9650,25.0 37 - 8192,3,8,4096,,true,10,9848434052,9392,24.6 38 - 8192,3,8,4096,4,true,10,10040294086,9575,16.8 39 - 8192,3,8,4096,16,true,10,9717871364,9267,22.5 40 - 8192,3,8,4096,64,true,10,9665939593,9218,25.4 41 - 8192,3,8,16384,,true,10,9859705853,9402,25.6 42 - 8192,3,8,16384,4,true,10,9782101780,9328,24.5 43 - 8192,3,8,16384,16,true,10,9713733091,9263,26.0 44 - 8192,3,8,16384,64,true,10,9694061307,9244,27.5 45 - 8192,3,8,65536,,true,10,9849121697,9392,26.2 46 - 8192,3,8,65536,4,true,10,9745808012,9294,24.9 47 - 8192,3,8,65536,16,true,10,9696039387,9246,27.5 48 - 8192,3,8,65536,64,true,10,9672733565,9224,31.5 49 - 8192,6,8,0,,true,10,9874369448,9416,31.7 50 - 8192,6,8,16384,16,true,10,9555000218,9112,38.5 51 - 16384,1,8,0,,true,10,9401904407,8966,15.6 52 - 16384,1,8,16384,16,true,10,9384259110,8949,15.2 53 - 16384,3,8,0,,true,4,9439746342,9002,22.6 54 - 16384,3,8,0,,true,10,9439798265,9002,14.1 55 - 16384,3,8,16384,16,true,10,9336363403,8903,21.6 56 - 16384,6,8,0,,true,10,9238560723,8810,28.8 57 - 16384,6,8,16384,16,true,10,9157064709,8732,35.4 58 - 16384,9,8,0,,true,10,9227102912,8799,55.0 59 - 16384,9,8,16384,16,true,10,9002507344,8585,48.0 60 - 32768,1,8,0,,true,10,9161174067,8736,12.6 61 - 32768,1,8,16384,16,true,10,9129717484,8706,13.8 62 - 32768,3,4,0,,true,10,9186445992,8760,13.7 63 - 32768,3,8,0,,false,10,9074744264,8654,13.3 64 - 32768,3,8,0,,true,10,9074587560,8654,13.3 65 - 32768,3,8,16384,16,true,10,9060352672,8640,22.0 66 - 32768,3,16,0,,true,10,9009532082,8592,13.8 67 - 32768,3,32,0,,true,10,8974169294,8558,13.3 68 - 32768,3,64,0,,true,10,8953489124,8538,13.3 69 - 32768,6,8,0,,true,10,8959226119,8544,27.5 70 - 32768,6,8,16384,16,true,10,8891715689,8479,34.9 71 - 65536,1,8,0,,true,10,8940392508,8526,11.5 72 - 65536,1,8,16384,16,true,10,8940404881,8526,10.8 73 - 65536,3,8,0,,true,10,8861096723,8450,12.9 74 - 65536,3,8,16384,16,true,10,8861073512,8450,13.4 75 - 65536,6,8,0,,true,10,8742392375,8337,27.9 76 - 65536,6,8,16384,16,true,10,8742335065,8337,29.1 77 - 65536,9,8,0,,true,10,8633639539,8233,36.6 78 - 65536,9,8,16384,16,true,10,8633630034,8233,37.0 79 - 8192,9,8,0,,true,10,9869739461,9412,50.2 80 - 8192,9,8,16384,16,true,10,9399964893,8964,54.3 81 - 32768,9,8,0,,true,10,8857646963,8447,42.4 82 - 32768,9,8,16384,16,true,10,8747823404,8342,51.4 83 - 16384,3,8,0,,true,1,9439775263,9002,81.6 84 - 4096,0,8,0,,true,10,15722034231,14993,13.5 85 - 8192,0,8,0,,true,10,14715967944,14034,12.8 86 - 16384,0,8,0,,true,10,14045987598,13395,11.7 87 - 32768,0,8,0,,true,10,13597519906,12967,12.9 88 - 65536,0,8,0,,true,10,13332609861,12714,16.7 89 - 4096,3,8,1024,8,true,10,11071194210,10558,16.7 90 - 4096,3,8,1024,32,true,10,10803699447,10303,27.1 91 - 4096,3,8,2048,,true,10,10618567648,10126,30.1 92 - 4096,3,8,2048,4,true,10,11070980059,10558,21.3 93 - 4096,3,8,2048,8,true,10,11070998773,10558,21.2 94 - 4096,3,8,2048,16,true,10,10344211112,9865,25.7 95 - 4096,3,8,2048,32,true,10,10310192553,9832,29.4 96 - 4096,3,8,2048,64,true,10,10275154282,9799,29.1 97 - 4096,3,8,4096,8,true,10,10323537489,9845,30.1 98 - 4096,3,8,4096,32,true,10,10235055391,9760,29.2 99 - 4096,3,8,8192,,true,10,10554955403,10065,30.6 100 - 4096,3,8,8192,4,true,10,10343018691,9863,26.8 101 - 4096,3,8,8192,8,true,10,10259861034,9784,29.0 102 - 4096,3,8,8192,16,true,10,10226087656,9752,29.2 103 - 4096,3,8,8192,32,true,10,10208952277,9736,29.4 104 - 4096,3,8,8192,64,true,10,10201640423,9729,29.8 105 - 4096,3,8,16384,8,true,10,10249137059,9774,30.1 106 - 4096,3,8,16384,32,true,10,10224396141,9750,30.7 107 - 4096,3,8,65536,8,true,10,10187080812,9715,30.9 108 - 4096,3,8,65536,32,true,10,10154462199,9684,33.1 109 - 8192,3,8,1024,8,true,10,10040350281,9575,22.2 110 - 8192,3,8,1024,32,true,10,10040330045,9575,16.4 111 - 8192,3,8,2048,,true,10,9829590607,9374,23.4 112 - 8192,3,8,2048,4,true,10,10040326898,9575,16.5 113 - 8192,3,8,2048,8,true,10,10040331586,9575,16.5 114 - 8192,3,8,2048,16,true,10,10040326454,9575,16.5 115 - 8192,3,8,2048,32,true,10,10107002899,9638,23.1 116 - 8192,3,8,2048,64,true,10,10089763846,9622,24.7 117 - 8192,3,8,4096,8,true,10,10040335592,9575,18.1 118 - 8192,3,8,4096,32,true,10,9694641840,9245,22.3 119 - 8192,3,8,8192,,true,10,9844667804,9388,24.5 120 - 8192,3,8,8192,4,true,10,10040337106,9575,16.8 121 - 8192,3,8,8192,8,true,10,9746958438,9295,23.3 122 - 8192,3,8,8192,16,true,10,9708134370,9258,25.7 123 - 8192,3,8,8192,32,true,10,9682607740,9234,25.7 124 - 8192,3,8,8192,64,true,10,9674036954,9225,26.2 125 - 8192,3,8,16384,8,true,10,9737625952,9286,26.8 126 - 8192,3,8,16384,32,true,10,9699236736,9249,26.5 127 - 8192,3,8,65536,8,true,10,9710583278,9260,26.5 128 - 8192,3,8,65536,32,true,10,9683941181,9235,28.6 129 - 131072,0,8,0,,true,10,13201586596,12590,13.5 130 - 131072,1,8,0,,true,10,8823092338,8414,10.4 131 - 131072,1,8,16384,16,true,10,8823145617,8414,10.1 132 - 131072,3,8,0,,true,10,8712125430,8308,12.5 133 - 131072,3,8,16384,16,true,10,8712317883,8308,12.9 134 - 131072,6,8,0,,true,10,8600177778,8201,27.0 135 - 131072,6,8,16384,16,true,10,8600107633,8201,27.6 136 - 131072,9,8,0,,true,10,8487562590,8094,38.2 137 - 131072,9,8,16384,16,true,10,8487490042,8094,38.8 2 + 4096,0,8,0,,true,10,31395311799,29940,19.8 3 + 4096,1,8,0,,true,10,22116416091,21091,30.7 4 + 4096,1,8,16384,16,true,10,20659329178,19702,42.5 5 + 4096,3,4,0,,true,10,22374836755,21338,51.4 6 + 4096,3,8,0,,true,10,22100643559,21076,41.7 7 + 4096,3,8,1024,,true,10,21177970088,20196,57.0 8 + 4096,3,8,1024,4,true,10,22100668085,21076,42.2 9 + 4096,3,8,1024,8,true,10,22100676471,21076,43.2 10 + 4096,3,8,1024,16,true,10,22100669290,21076,42.9 11 + 4096,3,8,1024,32,true,10,21526349249,20529,55.1 12 + 4096,3,8,1024,64,true,10,21501054074,20505,60.0 13 + 4096,3,8,2048,,true,10,21184609495,20203,59.8 14 + 4096,3,8,2048,4,true,10,22100673247,21076,42.1 15 + 4096,3,8,2048,8,true,10,22100621663,21076,41.2 16 + 4096,3,8,2048,16,true,10,20635793718,19679,52.0 17 + 4096,3,8,2048,32,true,10,20562444599,19609,58.2 18 + 4096,3,8,2048,64,true,10,20508962586,19558,57.6 19 + 4096,3,8,4096,,true,10,21128768715,20149,61.0 20 + 4096,3,8,4096,4,true,10,22100685331,21076,41.4 21 + 4096,3,8,4096,8,true,10,20577552594,19624,51.9 22 + 4096,3,8,4096,16,true,10,20479709404,19530,60.2 23 + 4096,3,8,4096,32,true,10,20403721320,19458,60.9 24 + 4096,3,8,4096,64,true,10,20368652511,19425,62.7 25 + 4096,3,8,8192,,true,10,21057872538,20082,62.9 26 + 4096,3,8,8192,4,true,10,20632375521,19676,54.1 27 + 4096,3,8,8192,8,true,10,20506735960,19556,61.2 28 + 4096,3,8,8192,16,true,10,20390435178,19445,62.1 29 + 4096,3,8,8192,32,true,10,20363765340,19420,61.9 30 + 4096,3,8,8192,64,true,10,20331974936,19390,61.6 31 + 4096,3,8,16384,,true,10,21083190741,20106,65.0 32 + 4096,3,8,16384,4,true,10,20579253867,19625,54.2 33 + 4096,3,8,16384,8,true,10,20455471802,19507,61.6 34 + 4096,3,8,16384,16,true,10,20405638509,19460,63.1 35 + 4096,3,8,16384,32,true,10,20386866978,19442,65.3 36 + 4096,3,8,16384,64,true,10,20367953764,19424,67.5 37 + 4096,3,8,65536,,true,10,20893557686,19925,63.9 38 + 4096,3,8,65536,4,true,10,20409555824,19464,56.4 39 + 4096,3,8,65536,8,true,10,20345337995,19402,64.1 40 + 4096,3,8,65536,16,true,10,20298445731,19358,65.1 41 + 4096,3,8,65536,32,true,10,20261884271,19323,69.0 42 + 4096,3,8,65536,64,true,10,20239324109,19301,74.3 43 + 4096,3,16,0,,true,10,21909626509,20894,48.0 44 + 4096,3,32,0,,true,10,21823083686,20812,39.5 45 + 4096,3,64,0,,true,10,21823091636,20812,40.4 46 + 4096,6,8,0,,true,10,21750720790,20743,65.2 47 + 4096,6,8,16384,16,true,10,20146634422,19213,84.1 48 + 4096,9,8,0,,true,10,21746294515,20738,111.8 49 + 4096,9,8,16384,16,true,10,19849302063,18929,116.6 50 + 8192,0,8,0,,true,10,29379196929,28018,35.0 51 + 8192,1,8,0,,true,10,20005967746,19079,25.5 52 + 8192,1,8,16384,16,true,10,19572338449,18665,34.0 53 + 8192,3,8,0,,false,10,20031898121,19103,36.1 54 + 8192,3,8,0,,true,10,20031920603,19103,31.9 55 + 8192,3,8,1024,,true,10,19622953913,18713,48.9 56 + 8192,3,8,1024,4,true,10,20031917101,19103,33.8 57 + 8192,3,8,1024,8,true,10,20031918966,19103,33.5 58 + 8192,3,8,1024,16,true,10,20031904492,19103,33.5 59 + 8192,3,8,1024,32,true,10,20031915783,19103,33.1 60 + 8192,3,8,1024,64,true,10,20182475006,19247,50.1 61 + 8192,3,8,2048,,true,10,19620130060,18711,52.0 62 + 8192,3,8,2048,4,true,10,20031954949,19103,35.7 63 + 8192,3,8,2048,8,true,10,20031939452,19103,33.7 64 + 8192,3,8,2048,16,true,10,20031949661,19103,33.4 65 + 8192,3,8,2048,32,true,10,20143626622,19210,47.8 66 + 8192,3,8,2048,64,true,10,20120486239,19188,52.3 67 + 8192,3,8,4096,,true,10,19635342683,18725,48.9 68 + 8192,3,8,4096,4,true,10,20031954812,19103,33.2 69 + 8192,3,8,4096,8,true,10,20031942177,19103,33.3 70 + 8192,3,8,4096,16,true,10,19372492949,18475,47.6 71 + 8192,3,8,4096,32,true,10,19313920339,18419,50.4 72 + 8192,3,8,4096,64,true,10,19282505585,18389,52.1 73 + 8192,3,8,8192,,true,10,19635635166,18726,50.9 74 + 8192,3,8,8192,4,true,10,20031928773,19103,35.0 75 + 8192,3,8,8192,8,true,10,19410703858,18511,46.3 76 + 8192,3,8,8192,16,true,10,19341160203,18445,50.9 77 + 8192,3,8,8192,32,true,10,19296132462,18402,50.9 78 + 8192,3,8,8192,64,true,10,19273631354,18380,51.8 79 + 8192,3,8,16384,,true,10,19655701449,18745,54.3 80 + 8192,3,8,16384,4,true,10,19502525484,18599,50.5 81 + 8192,3,8,16384,8,true,10,19419249737,18519,51.3 82 + 8192,3,8,16384,16,true,10,19358437498,18461,52.4 83 + 8192,3,8,16384,32,true,10,19340825956,18444,53.1 84 + 8192,3,8,16384,64,true,10,19329911907,18434,54.7 85 + 8192,3,8,65536,,true,10,19654121838,18743,52.6 86 + 8192,3,8,65536,4,true,10,19436643569,18536,50.1 87 + 8192,3,8,65536,8,true,10,19376580371,18478,54.5 88 + 8192,3,8,65536,16,true,10,19337019622,18441,55.6 89 + 8192,3,8,65536,32,true,10,19318086405,18423,59.0 90 + 8192,3,8,65536,64,true,10,19305582440,18411,64.8 91 + 8192,6,8,0,,true,10,19702719743,18789,62.2 92 + 8192,6,8,16384,16,true,10,19065986792,18182,74.2 93 + 8192,9,8,0,,true,10,19693493308,18781,100.9 94 + 8192,9,8,16384,16,true,10,18747260904,17878,100.7 95 + 16384,0,8,0,,true,10,28042902316,26743,26.0 96 + 16384,1,8,0,,true,10,18751959286,17883,22.9 97 + 16384,1,8,16384,16,true,10,18714866778,17847,30.2 98 + 16384,3,8,0,,true,1,18829928785,17957,174.2 99 + 16384,3,8,0,,true,4,18829973330,17957,44.1 100 + 16384,3,8,0,,true,10,18830038815,17957,27.8 101 + 16384,3,8,16384,16,true,10,18615498021,17753,43.5 102 + 16384,6,8,0,,true,10,18431835527,17577,57.9 103 + 16384,6,8,16384,16,true,10,18267813872,17421,69.3 104 + 16384,9,8,0,,true,10,18408979927,17556,109.1 105 + 16384,9,8,16384,16,true,10,17956014400,17124,95.6 106 + 32768,0,8,0,,true,10,27149041457,25891,22.3 107 + 32768,1,8,0,,true,10,18270567659,17424,21.6 108 + 32768,1,8,16384,16,true,10,18211125411,17367,27.5 109 + 32768,3,4,0,,true,10,18324784539,17475,28.5 110 + 32768,3,8,0,,false,10,18101433642,17262,27.5 111 + 32768,3,8,0,,true,10,18101394905,17262,26.5 112 + 32768,3,8,16384,16,true,10,18076847819,17239,46.5 113 + 32768,3,16,0,,true,10,17971384889,17138,28.1 114 + 32768,3,32,0,,true,10,17900644184,17071,27.1 115 + 32768,3,64,0,,true,10,17859565644,17032,29.2 116 + 32768,6,8,0,,true,10,17875945091,17047,56.9 117 + 32768,6,8,16384,16,true,10,17743235923,16921,72.5 118 + 32768,9,8,0,,true,10,17672322648,16853,76.0 119 + 32768,9,8,16384,16,true,10,17455398843,16646,96.0 120 + 65536,0,8,0,,true,10,26622152522,25388,21.2 121 + 65536,1,8,0,,true,10,17829479045,17003,21.0 122 + 65536,1,8,16384,16,true,10,17829619661,17003,21.4 123 + 65536,3,8,0,,true,10,17676395890,16857,25.5 124 + 65536,3,8,16384,16,true,10,17676353730,16857,26.4 125 + 65536,6,8,0,,true,10,17445895583,16637,53.6 126 + 65536,6,8,16384,16,true,10,17445851139,16637,54.6 127 + 65536,9,8,0,,true,10,17228573735,16430,73.1 128 + 65536,9,8,16384,16,true,10,17228513145,16430,73.9 129 + 131072,0,8,0,,true,10,26362266962,25141,19.5 130 + 131072,1,8,0,,true,10,17595617600,16780,20.4 131 + 131072,1,8,16384,16,true,10,17595914003,16780,20.0 132 + 131072,3,8,0,,true,10,17381327820,16576,24.9 133 + 131072,3,8,16384,16,true,10,17381302914,16576,25.5 134 + 131072,6,8,0,,true,10,17164905178,16369,54.9 135 + 131072,6,8,16384,16,true,10,17165045658,16369,56.9 136 + 131072,9,8,0,,true,10,16940090482,16155,76.5 137 + 131072,9,8,16384,16,true,10,16940006516,16155,76.7

Configure Feed

Configure Feed