my over complex system configurations
dotfiles.isabelroses.com/
nixos
nix
flake
dotfiles
linux
1name: Lix Diff
2
3on:
4 pull_request:
5
6permissions: {}
7
8jobs:
9 lix-diff:
10 runs-on: ubuntu-latest
11 permissions:
12 contents: read
13 pull-requests: write
14
15 steps:
16 - name: Checkout code
17 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
18 with:
19 persist-credentials: false
20
21 - name: Install Lix
22 uses: samueldr/lix-gha-installer-action@7b7f14d320d6aacfb65bd1ef761566b3b69e474c # v2026-02-22
23 with:
24 extra_nix_config: |
25 substituters = https://cache.nixos.org/ https://nix-community.cachix.org https://isabelroses.cachix.org https://catppuccin.cachix.org https://extersia.cachix.org
26 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= isabelroses.cachix.org-1:mXdV/CMcPDaiTmkQ7/4+MzChpOe6Cb97njKmBQQmLPM= catppuccin.cachix.org-1:noG/4HkbhJb+lUAdKrph6LaozJvAeEEZj4N732IysmU= extersia.cachix.org-1:ZHy9765xrhn4lDKGTzWWykHC+B091oTqNxClgc78MQU=
27
28 - name: Get Hosts
29 id: hosts
30 run: |
31 set -euo pipefail
32 {
33 echo "attributes<<EOF"
34 # shellcheck disable=SC2016
35 nix eval --impure --json --expr '
36 let
37 flake = (builtins.getFlake (toString ./.)).outputs;
38 mk = attr: toplevel:
39 map (name: {
40 displayName = name;
41 attribute = "${attr}.${name}.${toplevel}";
42 }) (builtins.attrNames (flake.${attr} or {}));
43 in
44 mk "nixosConfigurations" "config.system.build.toplevel"
45 ++ mk "darwinConfigurations" "system"
46 '
47 echo "EOF"
48 } >> "$GITHUB_OUTPUT"
49
50 - name: Run lix-diff
51 uses: isabelroses/lix-diff-action@813069195be03ebafecaf0ad00823b853baf1057 # main
52 with:
53 attributes: ${{ steps.hosts.outputs.attributes }}
54 comment-strategy: update
55
56 nix-eval-stats:
57 runs-on: ubuntu-latest
58 permissions:
59 contents: read
60 pull-requests: write
61
62 steps:
63 - name: Checkout PR
64 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
65 with:
66 persist-credentials: false
67
68 - name: Install Lix
69 uses: samueldr/lix-gha-installer-action@7b7f14d320d6aacfb65bd1ef761566b3b69e474c # v2026-02-22
70
71 - name: Checkout base
72 uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
73 with:
74 ref: ${{ github.event.pull_request.base.sha }}
75 path: base
76 persist-credentials: false
77
78 - name: Eval stats (before)
79 working-directory: base
80 run: |
81 attr=.#nixosConfigurations.amaterasu.config.system.build.toplevel
82 nix eval "$attr" > /dev/null
83 for i in $(seq 1 5); do
84 NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH="../stats-before-$i.json" \
85 nix eval --no-eval-cache "$attr" > /dev/null
86 done
87
88 - name: Eval stats (after)
89 run: |
90 attr=.#nixosConfigurations.amaterasu.config.system.build.toplevel
91 nix eval "$attr" > /dev/null
92 for i in $(seq 1 5); do
93 NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH="stats-after-$i.json" \
94 nix eval --no-eval-cache "$attr" > /dev/null
95 done
96
97 - name: Generate stats table
98 run: |
99 python3 << 'PYEOF'
100 import json
101 import statistics
102
103 def flatten(d, prefix=''):
104 result = {}
105 for k, v in d.items():
106 if isinstance(v, dict):
107 result.update(flatten(v, f'{prefix}{k}.'))
108 elif isinstance(v, (int, float)):
109 result[f'{prefix}{k}'] = v
110 return result
111
112 def load_runs(pattern, count=5):
113 runs = []
114 for i in range(1, count + 1):
115 with open(pattern.format(i)) as f:
116 runs.append(flatten(json.load(f)))
117 return runs
118
119 def average_runs(runs):
120 all_keys = set()
121 for r in runs:
122 all_keys.update(r.keys())
123 result = {}
124 for k in all_keys:
125 values = [r[k] for r in runs if k in r]
126 result[k] = statistics.mean(values)
127 return result
128
129 def stddev_runs(runs):
130 all_keys = set()
131 for r in runs:
132 all_keys.update(r.keys())
133 result = {}
134 for k in all_keys:
135 values = [r[k] for r in runs if k in r]
136 result[k] = statistics.stdev(values) if len(values) > 1 else 0
137 return result
138
139 before_runs = load_runs('stats-before-{}.json')
140 after_runs = load_runs('stats-after-{}.json')
141
142 before = average_runs(before_runs)
143 after = average_runs(after_runs)
144 before_sd = stddev_runs(before_runs)
145 after_sd = stddev_runs(after_runs)
146
147 all_keys = sorted(set(before) | set(after))
148
149 def fmt(n):
150 return f'{n:.3f}' if isinstance(n, float) else f'{int(n):,}'
151
152 def is_significant(before_runs, after_runs, key, threshold=0.05):
153 """Welch's t-test to determine if the difference is significant."""
154 import math
155 b_vals = [r[key] for r in before_runs if key in r]
156 a_vals = [r[key] for r in after_runs if key in r]
157 n_b, n_a = len(b_vals), len(a_vals)
158 if n_b < 2 or n_a < 2:
159 return False
160 mean_b = statistics.mean(b_vals)
161 mean_a = statistics.mean(a_vals)
162 var_b = statistics.variance(b_vals)
163 var_a = statistics.variance(a_vals)
164 se = math.sqrt(var_b / n_b + var_a / n_a)
165 if se == 0:
166 return mean_a != mean_b
167 t_stat = abs(mean_a - mean_b) / se
168 # Approximate p-value using degrees of freedom via Welch-Satterthwaite
169 num = (var_b / n_b + var_a / n_a) ** 2
170 denom = (var_b / n_b) ** 2 / (n_b - 1) + (var_a / n_a) ** 2 / (n_a - 1)
171 df = num / denom if denom > 0 else 1
172 # Conservative t-critical values for two-tailed p<0.05
173 # For df>=4 (our case with 5 runs each), t_crit ~ 2.78 (df=4) to 2.31 (df=8)
174 t_crit = 2.78 if df <= 4 else 2.45 if df <= 6 else 2.31
175 return t_stat > t_crit
176
177 lines = [
178 '| Metric | Before (mean +/- σ) | After (mean +/- σ) | Δ | % | Sig? |',
179 '|--------|---------------------|--------------------|----|---|------|',
180 ]
181 for key in all_keys:
182 b = before.get(key, 0)
183 a = after.get(key, 0)
184 bs = before_sd.get(key, 0)
185 as_ = after_sd.get(key, 0)
186 diff = a - b
187 pct = f'{diff / b * 100:+.1f}%' if b != 0 else 'N/A'
188 sign = '+' if diff > 0 else ''
189 sig = 'Yes' if is_significant(before_runs, after_runs, key) else ''
190 lines.append(f'| `{key}` | {fmt(b)} ± {fmt(bs)} | {fmt(a)} ± {fmt(as_)} | {sign}{fmt(diff)} | {pct} | {sig} |')
191
192 table = '\n'.join(lines)
193 with open('stats-table.md', 'w') as f:
194 f.write(f'## Nix Eval Stats: `amaterasu`\n\n{table}\n')
195 PYEOF
196
197 - name: Post comment
198 uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
199 with:
200 script: |
201 const fs = require('fs');
202 const body = fs.readFileSync('stats-table.md', 'utf8');
203 const marker = '## Nix Eval Stats: `amaterasu`';
204 const { data: comments } = await github.rest.issues.listComments({
205 owner: context.repo.owner,
206 repo: context.repo.repo,
207 issue_number: context.issue.number,
208 });
209 const existing = comments.find(c => c.body.includes(marker));
210 if (existing) {
211 await github.rest.issues.updateComment({
212 owner: context.repo.owner,
213 repo: context.repo.repo,
214 comment_id: existing.id,
215 body,
216 });
217 } else {
218 await github.rest.issues.createComment({
219 owner: context.repo.owner,
220 repo: context.repo.repo,
221 issue_number: context.issue.number,
222 body,
223 });
224 }
225