+2

bench/.gitignore

reviewed

··· 1 1 + *.json 2 2 + *.md

+4

bench/colmena/hive.nix

reviewed

··· 1 1 + let 2 2 + flake = import ../../default.nix; 3 3 + in 4 4 + import ../default.nix { inherit flake; }

+36

bench/default.nix

reviewed

··· 1 1 + { flake }: 2 2 + let 3 3 + nixpkgs = import flake.inputs.nixpkgs { }; 4 4 + 5 5 + vmNode = 6 6 + index: 7 7 + nixpkgs.lib.nameValuePair "bench-vm-${builtins.toString index}" { 8 8 + deployment = { 9 9 + targetPort = 2000 + index; 10 10 + targetHost = "localhost"; 11 11 + }; 12 12 + 13 13 + imports = [ 14 14 + ./vm.nix 15 15 + ]; 16 16 + 17 17 + _module.args = { 18 18 + index = builtins.toString index; 19 19 + }; 20 20 + 21 21 + deployment.keys = builtins.listToAttrs ( 22 22 + builtins.map ( 23 23 + index: 24 24 + nixpkgs.lib.nameValuePair "key-${builtins.toString index}" { 25 25 + keyFile = ./key.txt; 26 26 + } 27 27 + ) (nixpkgs.lib.range 0 200) 28 28 + ); 29 29 + 30 30 + nixpkgs.hostPlatform = "x86_64-linux"; 31 31 + }; 32 32 + in 33 33 + { 34 34 + meta.nixpkgs = nixpkgs; 35 35 + } 36 36 + // builtins.listToAttrs (builtins.map vmNode (nixpkgs.lib.range 0 20))

+1

bench/key.txt

reviewed

··· 1 1 + blixwqqujqfyfpsqcgnazvfmgoowggimroqngrbzvvawxprfrewrxgawcnkqqhlwkesyrpykabsshmyxjishnfphequzekyxtsacwqpruugcarendkmswnzercczxympaaddadqsxwtxbghvdjjtqrdfvmelnlloiczpafnccdognkfpugcbibutxsxxkucfdmewfqnqrjukoigckktqhdoaimhnpcpmdlkqkrsuakmyymzmoxxkwofpxgdpwxogjtrexratgzkgbrouiwuobefuweuhtlyyuigwniyqhvmsqxcxwcrbjgnmypbthuxzukkuiptbqhbgvbnktwthztrfjetthpguagswaqfpcykshvkzechxyruchddaqjribnajpvboopqnbstxqnuyawsrsqnljcwwzegcothixqltfmxssectaamgscngwejmadtmqclfuqukfxehwlitnhnhcfzsnandlyathmyxjvhxauvvjpkzxzkswgiyizkyvgmhlzildlxxrkejssslqodzjhbifddvqomqvbwyodnafluyctzyvnnpkslhnuhwnpjqnrtakqeezknstjjtijrnxsvbvwnmwjxoiwdysevzqvvbscjwvhvrzgmsosjbupchvsijhzemltvbpolsrgrcnlsvpqdxhrgmfhozzajavbshnzzdndcsjzvoahosxonpgragvqivolulvjybhnytnvrddicgdcdhbltlkbdzyyhltymvuhozkjrtsraucpufqpzkoexpsioagkblbouspmmwukeoprmbbbtpxlduxleaooaoynzfqmenxrpospatazsgsvghocochzqwaqzyypkarzuyhtgxlfoizntygobpbdikuhgmwupinslzazgbnngsekukaovfmhsvqgleulkywmxemebgyrilarghiyvaxwpdxkzpttrjkfrabdlqyqmrwyrvjqmtaghsoernqkbbhvkkzegdpadgoibladfjumvbckaxgleduswvxwproweukscpbjqnzibvpopariarmgwspfxfijljnkdmifynlpbcfbtuvxbupjuuzpaopadpszrltyrlotsdhiqduqqqdmnxxrzhtnwgenshdwnljcziylvmrjschfgifjumjrxyvfqgmywbtgpfnmnbhrjkbmczzabrbsqggcgsrvvlweejqyckvezcpqawiqvjxrdbrphcnakwjjgrgmonuyejnjojmvpghdhzmtmcdthnzgvbihsgowgelxbwmpevzcvkslmzcoqbmhtiyjjzjrfxgzoidqulslrfjnnbyzglvpulucpytanlyywptrwzvkrrzrzlgbhsjsqrvixrzkevfgxtoshzpugridbtyzporfburhwmccglhqdbgxqgqipuzefsrwhmtfmqdmmtcttcfpvqmxunhyofmktispmtgibgolhwaugvwtiiyrxlligmctgzhpajcwqsguoymxesdufzbivzvidnnccozkkyrqauxwknjarmtfgdeiqopozgxhlidoiyxdvigookwhukhnaizmxlfdwsioxgmgxegayzuxwanujnqfexhzqtgubqtbmozouighccucdgxteesnitceqhjxlemnoccxioalsuhlnchrhdejffupgkapwyhhynzmalsycgokkzckbufcegsjrnuuaxdqqhmuehftjbvemozitbtuleycbxcbavtqicsqgkmsclxqhzhqnsbzqbnbqwoehqyjnmrvowkfejcrrxnfqxgmnlfucruowgsoakjhrjnawlcnqkbyvlccymsyfaaoiyubzwrkcxzjxsvdtipwyuwzqdjminswqxlmymzijiyhmcgqvcyshzkjbhbwncqguehlacmmzlshxjfvvjftqlpqqdbdbrnxdutjeuzvzjveoxotpwfzysbvkavfkiyhnttfitpfqwniixyzapbwevyyyibjlzbscyppbmmldnzqesykqjjhwgpvlugsedjwjwxjdgzkhytwkhzsopdkznyxhwctwwqryqtdivpbhxmxnfcumowbfzoqpmtmdwwfgsezhdkldejqnwgululpnbwlrmtkdvyzzsmxakthtrnhwcttkkrzicxyslroovbptajumzzryjwqwgxfgfogfnhgawprylaypbkxxhejpdzxasgeypacicudzlacdzrxpqqxujnmxyossjfiglycnqutqfcifdsfbnewgemphzwflfpcdyusakecerfzlhsyqhmgklshdghnsxazjcfkbaciqobngxztbpyaeriolrzlcqevdkrcszjshkxtndetsbfoppqmmijlchvsdxzsouasnpgkovfykruteyykzsnkbesnjdreqfujyeawwwqvhboszotkiyrazknxrzymneedaalzibwpizlfqahaozrnydrcfgcklzpccychatkwcsbmwcbtwjexakjqygiwdjpbwrpzjntmtykdylfcqywotfsesesjlbmzanruxooslcybgbsoytcjlneacdyxbwdkhisoistamrqwaugebbzapujycvthhzrjaexwnsfnvdhzwaxawwqekwsuuacemehxmctvjefpdyzzhcxvmtivvdmspgxvvdmkersxfqfmegujbauernsckhqveqcgkkymgmicjgjdpuvpylqpzwcemawhgcxoebsvagwvwxytiqfnnbyfuzpindozdcgeriemnjptwgwpoyfibndyrimnhlxtpplkxzqflsxxihluqeiyxcxbvpsijtdvevswohsfjpvyhciwoljbslamdlxvabqnrxjdfxihderjllbxqflldskprfkyuwwyatkdkdznqdaaveujtmnivvfwgabpespniwljolxluiohgrcawiulcsctkihxjluryxfldnbioqghwzhrkcusdzzttmxvzbprtquymjomftiybhzuxcvqlbkaochlyyapjthrgcujxjpcogqblfpblydnreiolwrvxdczjefsurtylstjzcgemgliiyndsyotojyjopvmlzlogmiievgrzltdotktosehiexbqgfwsfrilqlhununycgzrrdhjnethzkjsaluhgflzxwvmxyiylulrbnqozlrsaytyvkngcxtokwzolnrfmmfpnnojqriwjtwmemsaberjjipwomuptonsfqiojqcczxunwoeqnvvzltqaptkjwgjwfilrptyyexbsabrugibvknhmydvknlbjamvwbfiiaemcaxqpmvujhrjqkeabbicrwtweyfwfapmnqinthozpkdiofukezcgfmtbspczfiervzeptdpnxfzzuiwtwlcbamlblcbjbtaljfvmyfbohtuvtttqdqmzchkmpzmtqzupzqqivjgcwlxxcuexykpslrylraxzsqrvbmfscsnsxaywzryskllgkyiudkhvxlajtonhukromstlnljtgaspwymmyunxxvigepcxwwlgxcmfwagvlnlgevdcbenfenogywefabmdbudpemiwvdxiqvncplehtbvrjkqloihgebbzshlnrjvbzqbbblkzzifaraviubwehgbjvzdrnyvpdvlyvmvfzzuwgfthresdrwyypmgnxexuhvsdhtdefqeyhfwtefkbxxaucezxsantnntnljvspsatbtdywjvhrxabxicsxqixypsliqofexlztcearqjruxndflxljrmkltbqvlwfeznttkeakdjhbdqzdyebfgwlgxnndsfsgzevhuaifikyqgvnndqtwpizwtazekveutwuwjkklwekkilcrsixzvqhceqiwcdqhdhpkqtrgwzztbwfvywicemwttlypyfwojluidadztbqqxvzfebdakwcquqfnnjelzjbnjmjqrfioiuhrqmcrmkcrjxzkymayunmxchbjeppfgugrsguamzbjuaebolaskixgddkuxfkredmtbvmfgijfckymxkfjoyriihizwvxkayruichaxxrmxqpvcwohcyhmgejjekybsleqepgmtqgrrsmwtqsdgtjaicaanwaggkukdhxadbnedgnilohnlbugmsbsjstqwwpsjqxyezoaeqitgdbtnlpnicqrhmlklokbhyurgdkflpvkwlhkkznqwsjjywesuopyylvjgzgfbsgwopjaiokdvfkgyzsjdtoytbxrinjazyhvytkpyrjybzgtknyddqywgyiwqtarpgmarpqnpndumisfzcmarhhqikmepfmpcvpfyougjujqjtfkbghwgxfrlklbiuqipppzfdlrnatenccjgssgszemfbtrguikiheejhgsrchjolozflgrrgnkzfwbspzabcsvarosquqhsqqxammqrqxklvazseeyzpofkkgpsmpetmwducilvwxoxnjhupccpvllajckrbhssktylvyffwoaaqajgupzkjaxedebwibkmpcudnkrageojafqrkbmydccghgmtnuuyiewasdoqzmfplspbqlhaoxvaypccwonetojkaasiiykeeckqmhanrfepbzdseycieuedvilmqwnteeqrseofabfvlpozbcedfuxpilrydpesnwclfzocfaehgphigecvraooopwtaxiysdhxhdoybsmanpckxhbqhzhcpnvcngdmfoquvrddhwoyuuprbqoillrvmxwyjvukjczzdqhlcrsssdpnvepbyirpxwymcqvzpmrddizjsrqmawdqhbymcdtfcjpbxlsqvlkyecroiivzqctvizzcadafztndwvfqrbrbntpbqcjlbwjuhlhwbeivdmimyeewizqtzlfwdcnsdfaupdewbgwfnkrzcunhobzjqgfppnuubcvbysbmrgjzjwhcfrxdodfsyijgprzfprryctwpawqdjpgljhrifspdbmczcrhlhwnbcchnxopkaksbcyxq

+50

bench/run.nix

reviewed

··· 1 1 + { inputs, ... }: 2 2 + { 3 3 + perSystem = 4 4 + { 5 5 + pkgs, 6 6 + lib, 7 7 + self', 8 8 + ... 9 9 + }: 10 10 + { 11 11 + packages = { 12 12 + bench-runner = pkgs.writeShellScriptBin "bench-runner" '' 13 13 + set -e 14 14 + 15 15 + export NIX_PATH="nixpkgs=${inputs.nixpkgs}" 16 16 + 17 17 + setup_vm() { 18 18 + local i="$1" 19 19 + 20 20 + echo "building $i" 21 21 + 22 22 + out=$(INDEX="$i" nix-build '<nixpkgs/nixos>' -A vm -I nixos-config=./vm.nix --no-link) 23 23 + 24 24 + echo "$out/bin/run-bench-vm-$i-vm" 25 25 + } 26 26 + 27 27 + export -f setup_vm 28 28 + 29 29 + echo "setting up vms..." 30 30 + ${lib.getExe pkgs.parallel-full} --verbose --tagstring {//} setup_vm {} ::: {0..20} | xargs -I {} bash -c '{} &' 31 31 + 32 32 + echo "sleeping" 33 33 + sleep 30 34 34 + echo "awake" 35 35 + 36 36 + wire_main=$(nix build --print-out-paths github:mrshmllow/wire#wire-small --no-link) 37 37 + wire_args="apply test --path ./wire -vv --ssh-accept-host -p 10" 38 38 + colmena_args="apply test --config ./colmena/hive.nix -v -p 10" 39 39 + 40 40 + ${lib.getExe pkgs.hyperfine} --warmup 1 --show-output --runs 1 \ 41 41 + --export-markdown stats.md \ 42 42 + --export-json run.json \ 43 43 + "${lib.getExe self'.packages.wire-small} $wire_args" -n "wire@HEAD" \ 44 44 + "$wire_main/bin/wire $wire_args" -n "wire@main" \ 45 45 + "${lib.getExe' inputs.colmena_benchmarking.packages.x86_64-linux.colmena "colmena"} $colmena_args" \ 46 46 + -n "colmena@pinned" 47 47 + ''; 48 48 + }; 49 49 + }; 50 50 + }

+58

bench/vm.nix

reviewed

··· 1 1 + { lib, index, ... }: 2 2 + let 3 3 + flake = import ../default.nix; 4 4 + in 5 5 + { 6 6 + _module.args = { 7 7 + index = lib.mkDefault (builtins.getEnv "INDEX"); 8 8 + }; 9 9 + 10 10 + imports = [ "${flake.inputs.nixpkgs}/nixos/modules/virtualisation/qemu-vm.nix" ]; 11 11 + 12 12 + networking.hostName = "bench-vm-${index}"; 13 13 + 14 14 + boot = { 15 15 + loader = { 16 16 + systemd-boot.enable = true; 17 17 + efi.canTouchEfiVariables = true; 18 18 + timeout = 0; 19 19 + }; 20 20 + 21 21 + kernelParams = [ "console=ttyS0" ]; 22 22 + }; 23 23 + 24 24 + services = { 25 25 + openssh = { 26 26 + enable = true; 27 27 + settings = { 28 28 + PermitRootLogin = "without-password"; 29 29 + }; 30 30 + }; 31 31 + 32 32 + getty.autologinUser = "root"; 33 33 + }; 34 34 + 35 35 + virtualisation = { 36 36 + graphics = false; 37 37 + # useBootLoader = true; 38 38 + 39 39 + diskSize = 5024; 40 40 + diskImage = null; 41 41 + 42 42 + forwardPorts = [ 43 43 + { 44 44 + from = "host"; 45 45 + host.port = 2000 + lib.toIntBase10 index; 46 46 + guest.port = 22; 47 47 + } 48 48 + ]; 49 49 + }; 50 50 + 51 51 + users.users.root.openssh.authorizedKeys.keys = [ 52 52 + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPSvOZoSGVEpR6eTDK9OJ31MWQPF2s8oLc8J7MBh6nez marsh@maple" 53 53 + ]; 54 54 + 55 55 + users.users.root.initialPassword = "root"; 56 56 + 57 57 + system.stateVersion = "23.11"; 58 58 + }

+5

bench/wire/hive.nix

reviewed

··· 1 1 + let 2 2 + flake = import ../../default.nix; 3 3 + wire = import flake; 4 4 + in 5 5 + wire.makeHive (import ../default.nix { inherit flake; })

+3 -5

default.nix

reviewed

··· 6 6 sha256 = flake-compat-node.locked.narHash; 7 7 }; 8 8 9 9 - flake = ( 10 10 - import flake-compat { 11 11 - src = ./.; 12 12 - } 13 13 - ); 9 9 + flake = import flake-compat { 10 10 + src = ./.; 11 11 + }; 14 12 in 15 13 flake.defaultNix

+97 -11

flake.lock

reviewed

··· 1 1 { 2 2 "nodes": { 3 3 + "colmena_benchmarking": { 4 4 + "inputs": { 5 5 + "flake-compat": "flake-compat", 6 6 + "flake-utils": "flake-utils", 7 7 + "nixpkgs": "nixpkgs", 8 8 + "stable": "stable" 9 9 + }, 10 10 + "locked": { 11 11 + "lastModified": 1684127108, 12 12 + "narHash": "sha256-01bfuSY4gnshhtqA1EJCw2CMsKkAx+dHS+sEpQ2+EAQ=", 13 13 + "owner": "zhaofengli", 14 14 + "repo": "colmena", 15 15 + "rev": "5fdd743a11e7291bd8ac1e169d62ba6156c99be4", 16 16 + "type": "github" 17 17 + }, 18 18 + "original": { 19 19 + "owner": "zhaofengli", 20 20 + "ref": "v0.4.0", 21 21 + "repo": "colmena", 22 22 + "type": "github" 23 23 + } 24 24 + }, 3 25 "crane": { 4 26 "locked": { 5 27 "lastModified": 1759893430, ··· 37 59 } 38 60 }, 39 61 "flake-compat": { 62 62 + "flake": false, 63 63 + "locked": { 64 64 + "lastModified": 1650374568, 65 65 + "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", 66 66 + "owner": "edolstra", 67 67 + "repo": "flake-compat", 68 68 + "rev": "b4a34015c698c7793d592d66adbab377907a2be8", 69 69 + "type": "github" 70 70 + }, 71 71 + "original": { 72 72 + "owner": "edolstra", 73 73 + "repo": "flake-compat", 74 74 + "type": "github" 75 75 + } 76 76 + }, 77 77 + "flake-compat_2": { 40 78 "locked": { 41 79 "lastModified": 1751685974, 42 80 "narHash": "sha256-NKw96t+BgHIYzHUjkTK95FqYRVKB8DHpVhefWSz/kTw=", ··· 49 87 "url": "https://git.lix.systems/lix-project/flake-compat/archive/main.tar.gz" 50 88 } 51 89 }, 52 52 - "flake-compat_2": { 90 90 + "flake-compat_3": { 53 91 "flake": false, 54 92 "locked": { 55 93 "lastModified": 1747046372, ··· 83 121 "type": "github" 84 122 } 85 123 }, 124 124 + "flake-utils": { 125 125 + "locked": { 126 126 + "lastModified": 1659877975, 127 127 + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", 128 128 + "owner": "numtide", 129 129 + "repo": "flake-utils", 130 130 + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", 131 131 + "type": "github" 132 132 + }, 133 133 + "original": { 134 134 + "owner": "numtide", 135 135 + "repo": "flake-utils", 136 136 + "type": "github" 137 137 + } 138 138 + }, 86 139 "git-hooks": { 87 140 "inputs": { 88 88 - "flake-compat": "flake-compat_2", 141 141 + "flake-compat": "flake-compat_3", 89 142 "gitignore": "gitignore", 90 90 - "nixpkgs": "nixpkgs" 143 143 + "nixpkgs": "nixpkgs_2" 91 144 }, 92 145 "locked": { 93 146 "lastModified": 1760392170, ··· 141 194 }, 142 195 "nixpkgs": { 143 196 "locked": { 144 144 - "lastModified": 1759070547, 145 145 - "narHash": "sha256-JVZl8NaVRYb0+381nl7LvPE+A774/dRpif01FKLrYFQ=", 197 197 + "lastModified": 1683408522, 198 198 + "narHash": "sha256-9kcPh6Uxo17a3kK3XCHhcWiV1Yu1kYj22RHiymUhMkU=", 146 199 "owner": "NixOS", 147 200 "repo": "nixpkgs", 148 148 - "rev": "647e5c14cbd5067f44ac86b74f014962df460840", 201 201 + "rev": "897876e4c484f1e8f92009fd11b7d988a121a4e7", 149 202 "type": "github" 150 203 }, 151 204 "original": { 152 205 "owner": "NixOS", 153 153 - "ref": "nixpkgs-unstable", 206 206 + "ref": "nixos-unstable", 154 207 "repo": "nixpkgs", 155 208 "type": "github" 156 209 } ··· 172 225 }, 173 226 "nixpkgs_2": { 174 227 "locked": { 228 228 + "lastModified": 1759070547, 229 229 + "narHash": "sha256-JVZl8NaVRYb0+381nl7LvPE+A774/dRpif01FKLrYFQ=", 230 230 + "owner": "NixOS", 231 231 + "repo": "nixpkgs", 232 232 + "rev": "647e5c14cbd5067f44ac86b74f014962df460840", 233 233 + "type": "github" 234 234 + }, 235 235 + "original": { 236 236 + "owner": "NixOS", 237 237 + "ref": "nixpkgs-unstable", 238 238 + "repo": "nixpkgs", 239 239 + "type": "github" 240 240 + } 241 241 + }, 242 242 + "nixpkgs_3": { 243 243 + "locked": { 175 244 "lastModified": 1760284886, 176 245 "narHash": "sha256-TK9Kr0BYBQ/1P5kAsnNQhmWWKgmZXwUQr4ZMjCzWf2c=", 177 246 "owner": "NixOS", ··· 186 255 "type": "github" 187 256 } 188 257 }, 189 189 - "nixpkgs_3": { 258 258 + "nixpkgs_4": { 190 259 "locked": { 191 260 "lastModified": 1754340878, 192 261 "narHash": "sha256-lgmUyVQL9tSnvvIvBp7x1euhkkCho7n3TMzgjdvgPoU=", ··· 220 289 }, 221 290 "root": { 222 291 "inputs": { 292 292 + "colmena_benchmarking": "colmena_benchmarking", 223 293 "crane": "crane", 224 294 "fenix": "fenix", 225 225 - "flake-compat": "flake-compat", 295 295 + "flake-compat": "flake-compat_2", 226 296 "flake-parts": "flake-parts", 227 297 "git-hooks": "git-hooks", 228 298 "linux-systems": "linux-systems", 229 229 - "nixpkgs": "nixpkgs_2", 299 299 + "nixpkgs": "nixpkgs_3", 230 300 "nixpkgs_current_stable": "nixpkgs_current_stable", 231 301 "systems": "systems", 232 302 "treefmt-nix": "treefmt-nix" ··· 249 319 "type": "github" 250 320 } 251 321 }, 322 322 + "stable": { 323 323 + "locked": { 324 324 + "lastModified": 1669735802, 325 325 + "narHash": "sha256-qtG/o/i5ZWZLmXw108N2aPiVsxOcidpHJYNkT45ry9Q=", 326 326 + "owner": "NixOS", 327 327 + "repo": "nixpkgs", 328 328 + "rev": "731cc710aeebecbf45a258e977e8b68350549522", 329 329 + "type": "github" 330 330 + }, 331 331 + "original": { 332 332 + "owner": "NixOS", 333 333 + "ref": "nixos-22.11", 334 334 + "repo": "nixpkgs", 335 335 + "type": "github" 336 336 + } 337 337 + }, 252 338 "systems": { 253 339 "locked": { 254 340 "lastModified": 1681028828, ··· 266 352 }, 267 353 "treefmt-nix": { 268 354 "inputs": { 269 269 - "nixpkgs": "nixpkgs_3" 355 355 + "nixpkgs": "nixpkgs_4" 270 356 }, 271 357 "locked": { 272 358 "lastModified": 1760120816,

+4

flake.nix

reviewed

··· 15 15 16 16 # testing inputs 17 17 nixpkgs_current_stable.url = "github:NixOS/nixpkgs/nixos-25.05"; 18 18 + 19 19 + # benchmarking 20 20 + colmena_benchmarking.url = "github:zhaofengli/colmena/v0.4.0"; 18 21 }; 19 22 outputs = 20 23 { ··· 38 41 ./wire/key_agent 39 42 ./doc 40 43 ./tests/nix 44 44 + ./bench/run.nix 41 45 ]; 42 46 systems = import systems; 43 47