my over complex system configurations
dotfiles.isabelroses.com/
nixos
nix
flake
dotfiles
linux
ci: run nix-eval-diff 5 times
+53
-14
+53
-14
.github/workflows/diff.yml
+53
-14
.github/workflows/diff.yml
···
82
82
- name: Eval stats (before)
83
83
working-directory: base
84
84
run: |
85
-
nix eval .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null
86
-
NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=../stats-before.json \
87
-
nix eval --no-eval-cache .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null
85
+
attr=.#nixosConfigurations.amaterasu.config.system.build.toplevel
86
+
nix eval "$attr" > /dev/null
87
+
for i in $(seq 1 5); do
88
+
NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH="../stats-before-$i.json" \
89
+
nix eval --no-eval-cache "$attr" > /dev/null
90
+
done
88
91
89
92
- name: Eval stats (after)
90
93
run: |
91
-
nix eval .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null
92
-
NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=stats-after.json \
93
-
nix eval --no-eval-cache .#nixosConfigurations.amaterasu.config.system.build.toplevel > /dev/null
94
+
attr=.#nixosConfigurations.amaterasu.config.system.build.toplevel
95
+
nix eval "$attr" > /dev/null
96
+
for i in $(seq 1 5); do
97
+
NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH="stats-after-$i.json" \
98
+
nix eval --no-eval-cache "$attr" > /dev/null
99
+
done
94
100
95
101
- name: Generate stats table
96
102
run: |
97
103
python3 << 'PYEOF'
98
104
import json
105
+
import statistics
99
106
100
107
def flatten(d, prefix=''):
101
108
result = {}
···
106
113
result[f'{prefix}{k}'] = v
107
114
return result
108
115
109
-
with open('stats-before.json') as f:
110
-
before = flatten(json.load(f))
111
-
with open('stats-after.json') as f:
112
-
after = flatten(json.load(f))
116
+
def load_runs(pattern, count=5):
117
+
runs = []
118
+
for i in range(1, count + 1):
119
+
with open(pattern.format(i)) as f:
120
+
runs.append(flatten(json.load(f)))
121
+
return runs
122
+
123
+
def average_runs(runs):
124
+
all_keys = set()
125
+
for r in runs:
126
+
all_keys.update(r.keys())
127
+
result = {}
128
+
for k in all_keys:
129
+
values = [r[k] for r in runs if k in r]
130
+
result[k] = statistics.mean(values)
131
+
return result
132
+
133
+
def stddev_runs(runs):
134
+
all_keys = set()
135
+
for r in runs:
136
+
all_keys.update(r.keys())
137
+
result = {}
138
+
for k in all_keys:
139
+
values = [r[k] for r in runs if k in r]
140
+
result[k] = statistics.stdev(values) if len(values) > 1 else 0
141
+
return result
142
+
143
+
before_runs = load_runs('stats-before-{}.json')
144
+
after_runs = load_runs('stats-after-{}.json')
145
+
146
+
before = average_runs(before_runs)
147
+
after = average_runs(after_runs)
148
+
before_sd = stddev_runs(before_runs)
149
+
after_sd = stddev_runs(after_runs)
113
150
114
151
all_keys = sorted(set(before) | set(after))
115
152
116
153
def fmt(n):
117
-
return f'{n:.3f}' if isinstance(n, float) else f'{n:,}'
154
+
return f'{n:.3f}' if isinstance(n, float) else f'{int(n):,}'
118
155
119
156
lines = [
120
-
'| Metric | Before | After | Δ | % |',
121
-
'|--------|--------|-------|---|---|',
157
+
'| Metric | Before (mean ± σ) | After (mean ± σ) | Δ | % |',
158
+
'|--------|-------------------|------------------|---|---|',
122
159
]
123
160
for key in all_keys:
124
161
b = before.get(key, 0)
125
162
a = after.get(key, 0)
163
+
bs = before_sd.get(key, 0)
164
+
as_ = after_sd.get(key, 0)
126
165
diff = a - b
127
166
pct = f'{diff / b * 100:+.1f}%' if b != 0 else 'N/A'
128
167
sign = '+' if diff > 0 else ''
129
-
lines.append(f'| `{key}` | {fmt(b)} | {fmt(a)} | {sign}{fmt(diff)} | {pct} |')
168
+
lines.append(f'| `{key}` | {fmt(b)} ± {fmt(bs)} | {fmt(a)} ± {fmt(as_)} | {sign}{fmt(diff)} | {pct} |')
130
169
131
170
table = '\n'.join(lines)
132
171
with open('stats-table.md', 'w') as f: