my over complex system configurations dotfiles.isabelroses.com/
nixos nix flake dotfiles linux
9
fork

Configure Feed

Select the types of activity you want to include in your feed.

ci: run nix-eval-diff 5 times

isabel 19b8dbbb 7b30bd58

+53 -14
+53 -14
.github/workflows/diff.yml
··· 82 82 - name: Eval stats (before) 83 83 working-directory: base 84 84 run: | 85 - nix eval .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null 86 - NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=../stats-before.json \ 87 - nix eval --no-eval-cache .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null 85 + attr=.#nixosConfigurations.amaterasu.config.system.build.toplevel 86 + nix eval "$attr" > /dev/null 87 + for i in $(seq 1 5); do 88 + NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH="../stats-before-$i.json" \ 89 + nix eval --no-eval-cache "$attr" > /dev/null 90 + done 88 91 89 92 - name: Eval stats (after) 90 93 run: | 91 - nix eval .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null 92 - NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=stats-after.json \ 93 - nix eval --no-eval-cache .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null 94 + attr=.#nixosConfigurations.amaterasu.config.system.build.toplevel 95 + nix eval "$attr" > /dev/null 96 + for i in $(seq 1 5); do 97 + NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH="stats-after-$i.json" \ 98 + nix eval --no-eval-cache "$attr" > /dev/null 99 + done 94 100 95 101 - name: Generate stats table 96 102 run: | 97 103 python3 << 'PYEOF' 98 104 import json 105 + import statistics 99 106 100 107 def flatten(d, prefix=''): 101 108 result = {} ··· 106 113 result[f'{prefix}{k}'] = v 107 114 return result 108 115 109 - with open('stats-before.json') as f: 110 - before = flatten(json.load(f)) 111 - with open('stats-after.json') as f: 112 - after = flatten(json.load(f)) 116 + def load_runs(pattern, count=5): 117 + runs = [] 118 + for i in range(1, count + 1): 119 + with open(pattern.format(i)) as f: 120 + runs.append(flatten(json.load(f))) 121 + return runs 122 + 123 + def average_runs(runs): 124 + all_keys = set() 125 + for r in runs: 126 + all_keys.update(r.keys()) 127 + result = {} 128 + for k in all_keys: 129 + values = [r[k] for r in runs if k in r] 130 + result[k] = statistics.mean(values) 131 + return result 132 + 133 + def stddev_runs(runs): 134 + all_keys = set() 135 + for r in runs: 136 + all_keys.update(r.keys()) 137 + result = {} 138 + for k in all_keys: 139 + values = [r[k] for r in runs if k in r] 140 + result[k] = statistics.stdev(values) if len(values) > 1 else 0 141 + return result 142 + 143 + before_runs = load_runs('stats-before-{}.json') 144 + after_runs = load_runs('stats-after-{}.json') 145 + 146 + before = average_runs(before_runs) 147 + after = average_runs(after_runs) 148 + before_sd = stddev_runs(before_runs) 149 + after_sd = stddev_runs(after_runs) 113 150 114 151 all_keys = sorted(set(before) | set(after)) 115 152 116 153 def fmt(n): 117 - return f'{n:.3f}' if isinstance(n, float) else f'{n:,}' 154 + return f'{n:.3f}' if isinstance(n, float) else f'{int(n):,}' 118 155 119 156 lines = [ 120 - '| Metric | Before | After | Δ | % |', 121 - '|--------|--------|-------|---|---|', 157 + '| Metric | Before (mean ± σ) | After (mean ± σ) | Δ | % |', 158 + '|--------|-------------------|------------------|---|---|', 122 159 ] 123 160 for key in all_keys: 124 161 b = before.get(key, 0) 125 162 a = after.get(key, 0) 163 + bs = before_sd.get(key, 0) 164 + as_ = after_sd.get(key, 0) 126 165 diff = a - b 127 166 pct = f'{diff / b * 100:+.1f}%' if b != 0 else 'N/A' 128 167 sign = '+' if diff > 0 else '' 129 - lines.append(f'| `{key}` | {fmt(b)} | {fmt(a)} | {sign}{fmt(diff)} | {pct} |') 168 + lines.append(f'| `{key}` | {fmt(b)} ± {fmt(bs)} | {fmt(a)} ± {fmt(as_)} | {sign}{fmt(diff)} | {pct} |') 130 169 131 170 table = '\n'.join(lines) 132 171 with open('stats-table.md', 'w') as f: