this repo has no description
tests: intial e2e test in QEMU
+540
-20
+4
-1
Makefile
+4
-1
Makefile
+96
-19
examples/flake.nix
+96
-19
examples/flake.nix
···
21
21
disko,
22
22
nixie,
23
23
}:
24
-
{
25
-
nixosConfigurations = {
26
-
installer = nixpkgs.lib.nixosSystem {
27
-
system = "x86_64-linux";
28
-
modules = [
29
-
./installer.nix
30
-
nixie.nixosModules.nixie-agent
31
-
];
32
-
};
33
-
machine1 = nixpkgs.lib.nixosSystem {
34
-
system = "x86_64-linux";
24
+
let
25
+
linuxSystem = "x86_64-linux";
26
+
hostSystems = [
27
+
"x86_64-linux"
28
+
"x86_64-darwin"
29
+
"aarch64-darwin"
30
+
];
31
+
32
+
e2eModule = "${nixie.outPath}/tests/e2e/module.nix";
33
+
e2eRunnerFor =
34
+
system:
35
+
let
36
+
pkgs = import nixpkgs { inherit system; };
37
+
in
38
+
if system == linuxSystem then
39
+
pkgs.writeShellApplication {
40
+
name = "nixie-e2e";
41
+
runtimeInputs = with pkgs; [
42
+
dnsmasq
43
+
iproute2
44
+
nix
45
+
nixos-anywhere
46
+
openssh
47
+
OVMF.fd
48
+
python3
49
+
qemu_kvm
50
+
tcpdump
51
+
];
52
+
text = ''
53
+
export EXAMPLES_FLAKE="${self.outPath}"
54
+
export NIXIE_BIN="${nixie.packages.${linuxSystem}.default}/bin/nixie"
55
+
export NIXIE_REPO="${nixie.outPath}"
56
+
export OVMF_CODE="${pkgs.OVMF.fd}/FV/OVMF_CODE.fd"
57
+
export OVMF_VARS="${pkgs.OVMF.fd}/FV/OVMF_VARS.fd"
58
+
exec python3 "${nixie.outPath}/tests/e2e/run.py" "$@"
59
+
'';
60
+
}
61
+
else
62
+
pkgs.writeShellApplication {
63
+
name = "nixie-e2e";
64
+
text = ''
65
+
echo "The nixie e2e harness currently requires a Linux host with root access."
66
+
echo "Run nix run ./examples#e2e on a Linux machine to execute it."
67
+
exit 1
68
+
'';
69
+
};
70
+
71
+
mkInstalledMachine =
72
+
hostName: extraModules:
73
+
nixpkgs.lib.nixosSystem {
74
+
system = linuxSystem;
35
75
modules = [
36
76
disko.nixosModules.disko
37
77
./configuration.nix
38
78
{
39
-
networking.hostName = "machine1";
79
+
networking.hostName = hostName;
40
80
}
81
+
] ++ extraModules;
82
+
};
83
+
in
84
+
{
85
+
nixosModules = {
86
+
commonMachine = import ./configuration.nix;
87
+
installer = import ./installer.nix;
88
+
e2e = import e2eModule;
89
+
};
90
+
91
+
apps = nixpkgs.lib.genAttrs hostSystems (system: {
92
+
e2e = {
93
+
type = "app";
94
+
program = "${e2eRunnerFor system}/bin/nixie-e2e";
95
+
};
96
+
});
97
+
98
+
packages = nixpkgs.lib.genAttrs hostSystems (system: {
99
+
e2e = e2eRunnerFor system;
100
+
});
101
+
102
+
devShells.${linuxSystem}.e2e =
103
+
let
104
+
pkgs = import nixpkgs { system = linuxSystem; };
105
+
in
106
+
pkgs.mkShell {
107
+
packages = with pkgs; [
108
+
dnsmasq
109
+
iproute2
110
+
nix
111
+
nixos-anywhere
112
+
openssh
113
+
OVMF.fd
114
+
python3
115
+
qemu_kvm
116
+
tcpdump
41
117
];
42
118
};
43
-
machine2 = nixpkgs.lib.nixosSystem {
44
-
system = "x86_64-linux";
119
+
120
+
nixosConfigurations = {
121
+
installer = nixpkgs.lib.nixosSystem {
122
+
system = linuxSystem;
45
123
modules = [
46
-
disko.nixosModules.disko
47
-
./configuration.nix
48
-
{
49
-
networking.hostName = "machine2";
50
-
}
124
+
self.nixosModules.installer
125
+
nixie.nixosModules.nixie-agent
51
126
];
52
127
};
128
+
machine1 = mkInstalledMachine "machine1" [ ];
129
+
machine2 = mkInstalledMachine "machine2" [ ];
53
130
};
54
131
};
55
132
}
+8
tests/e2e/hosts.json
+8
tests/e2e/hosts.json
+19
tests/e2e/module.nix
+19
tests/e2e/module.nix
···
1
+
{
2
+
...
3
+
}:
4
+
5
+
{
6
+
boot.kernelParams = [
7
+
"console=tty0"
8
+
"console=ttyS0,115200n8"
9
+
];
10
+
11
+
services.openssh.settings = {
12
+
PasswordAuthentication = false;
13
+
PermitRootLogin = "yes";
14
+
};
15
+
16
+
systemd.services."serial-getty@ttyS0".enable = true;
17
+
18
+
environment.etc."nixie-e2e".text = "ready\n";
19
+
}
+411
tests/e2e/run.py
+411
tests/e2e/run.py
···
1
+
#!/usr/bin/env python3
2
+
3
+
import os
4
+
import secrets
5
+
import shlex
6
+
import shutil
7
+
import signal
8
+
import socket
9
+
import subprocess
10
+
import sys
11
+
import tempfile
12
+
import time
13
+
from pathlib import Path
14
+
15
+
16
+
NETWORK = {
17
+
"controller_ip": "192.168.50.2",
18
+
"cidr": "24",
19
+
"dhcp_range_start": "192.168.50.10",
20
+
"dhcp_range_end": "192.168.50.20",
21
+
"machine1_ip": "192.168.50.11",
22
+
"machine2_ip": "192.168.50.12",
23
+
}
24
+
25
+
MACHINES = [
26
+
{
27
+
"name": "machine1",
28
+
"mac": "52:54:00:12:34:56",
29
+
"ip": NETWORK["machine1_ip"],
30
+
},
31
+
{
32
+
"name": "machine2",
33
+
"mac": "52:54:00:12:34:57",
34
+
"ip": NETWORK["machine2_ip"],
35
+
},
36
+
]
37
+
38
+
39
+
def log(message):
40
+
print(f"[nixie-e2e] {message}", flush=True)
41
+
42
+
43
+
def run(cmd, **kwargs):
44
+
log(f"+ {' '.join(shlex.quote(part) for part in cmd)}")
45
+
subprocess.run(cmd, check=True, **kwargs)
46
+
47
+
48
+
def start_process(name, cmd, log_path, env=None):
49
+
log(f"starting {name}: {' '.join(shlex.quote(part) for part in cmd)}")
50
+
handle = open(log_path, "w", encoding="utf-8")
51
+
proc = subprocess.Popen(
52
+
cmd,
53
+
stdout=handle,
54
+
stderr=subprocess.STDOUT,
55
+
env=env,
56
+
start_new_session=True,
57
+
text=True,
58
+
)
59
+
return proc, handle
60
+
61
+
62
+
def kill_process(proc):
63
+
if proc.poll() is not None:
64
+
return
65
+
os.killpg(proc.pid, signal.SIGTERM)
66
+
try:
67
+
proc.wait(timeout=10)
68
+
except subprocess.TimeoutExpired:
69
+
os.killpg(proc.pid, signal.SIGKILL)
70
+
proc.wait(timeout=10)
71
+
72
+
73
+
def wait_for_port(host, port, timeout):
74
+
deadline = time.time() + timeout
75
+
while time.time() < deadline:
76
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
77
+
sock.settimeout(1)
78
+
try:
79
+
sock.connect((host, port))
80
+
return
81
+
except OSError:
82
+
time.sleep(1)
83
+
raise TimeoutError(f"timed out waiting for {host}:{port}")
84
+
85
+
86
+
def wait_for_nixie(proc, timeout):
87
+
deadline = time.time() + timeout
88
+
while time.time() < deadline:
89
+
rc = proc.poll()
90
+
if rc is not None:
91
+
if rc != 0:
92
+
raise RuntimeError(f"nixie exited with status {rc}")
93
+
return
94
+
time.sleep(5)
95
+
raise TimeoutError("timed out waiting for nixie to finish installing all hosts")
96
+
97
+
98
+
def wait_for_ssh(key_path, ip, expected_hostname, timeout):
99
+
deadline = time.time() + timeout
100
+
cmd = [
101
+
"ssh",
102
+
"-i",
103
+
str(key_path),
104
+
"-o",
105
+
"BatchMode=yes",
106
+
"-o",
107
+
"ConnectTimeout=5",
108
+
"-o",
109
+
"StrictHostKeyChecking=no",
110
+
"-o",
111
+
"UserKnownHostsFile=/dev/null",
112
+
f"root@{ip}",
113
+
"hostname && test -f /etc/nixie-e2e",
114
+
]
115
+
116
+
while time.time() < deadline:
117
+
result = subprocess.run(cmd, capture_output=True, text=True)
118
+
if result.returncode == 0:
119
+
hostname = result.stdout.splitlines()[0].strip()
120
+
if hostname == expected_hostname:
121
+
return
122
+
time.sleep(10)
123
+
124
+
raise TimeoutError(f"timed out waiting for SSH on {ip}")
125
+
126
+
127
+
def make_dnsmasq_config(path, bridge):
128
+
lines = [
129
+
"port=0",
130
+
"bind-interfaces",
131
+
"log-dhcp",
132
+
"dhcp-authoritative",
133
+
f"interface={bridge}",
134
+
f"dhcp-range={NETWORK['dhcp_range_start']},{NETWORK['dhcp_range_end']},255.255.255.0,1h",
135
+
f"dhcp-option=option:router,{NETWORK['controller_ip']}",
136
+
f"dhcp-option=option:dns-server,{NETWORK['controller_ip']}",
137
+
f"dhcp-host={MACHINES[0]['mac']},{NETWORK['machine1_ip']}",
138
+
f"dhcp-host={MACHINES[1]['mac']},{NETWORK['machine2_ip']}",
139
+
]
140
+
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
141
+
142
+
143
+
def tail(path):
144
+
if not path.exists():
145
+
return ""
146
+
text = path.read_text(encoding="utf-8", errors="replace")
147
+
lines = text.splitlines()
148
+
return "\n".join(lines[-40:])
149
+
150
+
151
+
def generate_ssh_keypair(workdir):
152
+
key_path = workdir / "id_ed25519"
153
+
run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key_path)])
154
+
pubkey = key_path.with_suffix(".pub").read_text(encoding="utf-8").strip()
155
+
return key_path, pubkey
156
+
157
+
158
+
def write_runtime_flake(workdir, pubkey):
159
+
flake_dir = workdir / "runtime-flake"
160
+
flake_dir.mkdir(parents=True, exist_ok=True)
161
+
162
+
key_module = flake_dir / "ssh-key.nix"
163
+
key_module.write_text(
164
+
"\n".join(
165
+
[
166
+
"{",
167
+
" ...",
168
+
"}:",
169
+
"",
170
+
"{",
171
+
" users.users.root.openssh.authorizedKeys.keys = [",
172
+
f' "{pubkey}"',
173
+
" ];",
174
+
"}",
175
+
"",
176
+
]
177
+
),
178
+
encoding="utf-8",
179
+
)
180
+
181
+
flake_nix = flake_dir / "flake.nix"
182
+
flake_nix.write_text(
183
+
f"""
184
+
{{
185
+
inputs = {{
186
+
nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05";
187
+
disko = {{
188
+
url = "github:nix-community/disko";
189
+
inputs.nixpkgs.follows = "nixpkgs";
190
+
}};
191
+
examples.url = "path:{os.environ["EXAMPLES_FLAKE"]}";
192
+
nixie = {{
193
+
url = "path:{os.environ["NIXIE_REPO"]}";
194
+
inputs.nixpkgs.follows = "nixpkgs";
195
+
}};
196
+
}};
197
+
198
+
outputs =
199
+
{{
200
+
nixpkgs,
201
+
disko,
202
+
examples,
203
+
nixie,
204
+
...
205
+
}}:
206
+
let
207
+
system = "x86_64-linux";
208
+
keyModule = import ./ssh-key.nix;
209
+
mkMachine =
210
+
hostName:
211
+
nixpkgs.lib.nixosSystem {{
212
+
inherit system;
213
+
modules = [
214
+
disko.nixosModules.disko
215
+
examples.nixosModules.commonMachine
216
+
{{
217
+
networking.hostName = hostName;
218
+
}}
219
+
examples.nixosModules.e2e
220
+
keyModule
221
+
];
222
+
}};
223
+
in
224
+
{{
225
+
nixosConfigurations = {{
226
+
e2e-installer = nixpkgs.lib.nixosSystem {{
227
+
inherit system;
228
+
modules = [
229
+
examples.nixosModules.installer
230
+
nixie.nixosModules.nixie-agent
231
+
examples.nixosModules.e2e
232
+
keyModule
233
+
];
234
+
}};
235
+
e2e-machine1 = mkMachine "machine1";
236
+
e2e-machine2 = mkMachine "machine2";
237
+
}};
238
+
}};
239
+
}}
240
+
""".strip()
241
+
+ "\n",
242
+
encoding="utf-8",
243
+
)
244
+
245
+
return flake_dir
246
+
247
+
248
+
def main():
249
+
if os.geteuid() != 0:
250
+
print("Run this as root, for example: sudo nix run ./examples#e2e", file=sys.stderr)
251
+
return 1
252
+
253
+
required_env = [
254
+
"EXAMPLES_FLAKE",
255
+
"NIXIE_BIN",
256
+
"NIXIE_REPO",
257
+
"OVMF_CODE",
258
+
"OVMF_VARS",
259
+
]
260
+
missing = [name for name in required_env if not os.environ.get(name)]
261
+
if missing:
262
+
print(f"missing required environment variables: {', '.join(missing)}", file=sys.stderr)
263
+
return 1
264
+
265
+
workdir_env = os.environ.get("NIXIE_E2E_WORKDIR")
266
+
if workdir_env:
267
+
workdir = Path(workdir_env).resolve()
268
+
workdir.mkdir(parents=True, exist_ok=True)
269
+
else:
270
+
workdir = Path(tempfile.mkdtemp(prefix="nixie-e2e-"))
271
+
272
+
log(f"artifacts directory: {workdir}")
273
+
274
+
suffix = secrets.token_hex(2)
275
+
bridge = f"brnxe{suffix}"
276
+
taps = [f"tapn1{suffix}", f"tapn2{suffix}"]
277
+
processes = []
278
+
279
+
key_path, pubkey = generate_ssh_keypair(workdir)
280
+
runtime_flake = write_runtime_flake(workdir, pubkey)
281
+
282
+
hosts_path = Path(os.environ["NIXIE_REPO"]) / "tests" / "e2e" / "hosts.json"
283
+
dnsmasq_conf = workdir / "dnsmasq.conf"
284
+
make_dnsmasq_config(dnsmasq_conf, bridge)
285
+
286
+
use_kvm = os.path.exists("/dev/kvm") and os.access("/dev/kvm", os.R_OK | os.W_OK)
287
+
if use_kvm:
288
+
log("using KVM acceleration")
289
+
else:
290
+
log("KVM is unavailable, falling back to software emulation")
291
+
292
+
try:
293
+
run(["ip", "link", "add", bridge, "type", "bridge"])
294
+
run(["ip", "addr", "add", f"{NETWORK['controller_ip']}/{NETWORK['cidr']}", "dev", bridge])
295
+
run(["ip", "link", "set", bridge, "up"])
296
+
297
+
for tap in taps:
298
+
run(["ip", "tuntap", "add", "dev", tap, "mode", "tap", "user", "root"])
299
+
run(["ip", "link", "set", tap, "master", bridge])
300
+
run(["ip", "link", "set", tap, "up"])
301
+
302
+
dnsmasq_log = workdir / "dnsmasq.log"
303
+
dnsmasq_proc, dnsmasq_handle = start_process(
304
+
"dnsmasq",
305
+
["dnsmasq", "--keep-in-foreground", f"--conf-file={dnsmasq_conf}"],
306
+
dnsmasq_log,
307
+
)
308
+
processes.append((dnsmasq_proc, dnsmasq_handle))
309
+
310
+
nixie_log = workdir / "nixie.log"
311
+
nixie_proc, nixie_handle = start_process(
312
+
"nixie",
313
+
[
314
+
os.environ["NIXIE_BIN"],
315
+
"--address",
316
+
NETWORK["controller_ip"],
317
+
"--installer",
318
+
f"{runtime_flake}#nixosConfigurations.e2e-installer",
319
+
"--flake",
320
+
str(runtime_flake),
321
+
"--hosts",
322
+
str(hosts_path),
323
+
"--debug",
324
+
],
325
+
nixie_log,
326
+
)
327
+
processes.append((nixie_proc, nixie_handle))
328
+
329
+
wait_for_port(NETWORK["controller_ip"], 5000, timeout=1800)
330
+
time.sleep(2)
331
+
332
+
for machine, tap in zip(MACHINES, taps, strict=True):
333
+
disk = workdir / f"{machine['name']}.qcow2"
334
+
vars_path = workdir / f"{machine['name']}-OVMF_VARS.fd"
335
+
serial_log = workdir / f"{machine['name']}.serial.log"
336
+
337
+
shutil.copyfile(os.environ["OVMF_VARS"], vars_path)
338
+
run(["qemu-img", "create", "-f", "qcow2", str(disk), "20G"])
339
+
340
+
cmd = [
341
+
"qemu-system-x86_64",
342
+
"-name",
343
+
machine["name"],
344
+
"-machine",
345
+
"q35",
346
+
"-m",
347
+
"2048",
348
+
"-smp",
349
+
"2",
350
+
"-display",
351
+
"none",
352
+
"-serial",
353
+
f"file:{serial_log}",
354
+
"-monitor",
355
+
"none",
356
+
"-drive",
357
+
f"if=pflash,format=raw,readonly=on,file={os.environ['OVMF_CODE']}",
358
+
"-drive",
359
+
f"if=pflash,format=raw,file={vars_path}",
360
+
"-drive",
361
+
f"if=none,id=disk0,file={disk},format=qcow2",
362
+
"-device",
363
+
"virtio-blk-pci,drive=disk0,bootindex=2",
364
+
"-netdev",
365
+
f"tap,id=net0,ifname={tap},script=no,downscript=no",
366
+
"-device",
367
+
f"e1000,netdev=net0,mac={machine['mac']},bootindex=1",
368
+
]
369
+
if use_kvm:
370
+
cmd.extend(["-enable-kvm", "-cpu", "host"])
371
+
else:
372
+
cmd.extend(["-cpu", "max"])
373
+
374
+
qemu_log = workdir / f"{machine['name']}.qemu.log"
375
+
qemu_proc, qemu_handle = start_process(machine["name"], cmd, qemu_log)
376
+
processes.append((qemu_proc, qemu_handle))
377
+
378
+
wait_for_nixie(nixie_proc, timeout=7200)
379
+
380
+
for machine in MACHINES:
381
+
wait_for_ssh(key_path, machine["ip"], machine["name"], timeout=1800)
382
+
log(f"verified {machine['name']} at {machine['ip']}")
383
+
384
+
log("end-to-end test passed")
385
+
return 0
386
+
except Exception as err:
387
+
log(f"end-to-end test failed: {err}")
388
+
print("\n== nixie.log ==", file=sys.stderr)
389
+
print(tail(workdir / "nixie.log"), file=sys.stderr)
390
+
print("\n== dnsmasq.log ==", file=sys.stderr)
391
+
print(tail(workdir / "dnsmasq.log"), file=sys.stderr)
392
+
for machine in MACHINES:
393
+
print(f"\n== {machine['name']}.serial.log ==", file=sys.stderr)
394
+
print(tail(workdir / f"{machine['name']}.serial.log"), file=sys.stderr)
395
+
return 1
396
+
finally:
397
+
for proc, handle in reversed(processes):
398
+
try:
399
+
kill_process(proc)
400
+
finally:
401
+
handle.close()
402
+
403
+
for tap in taps:
404
+
subprocess.run(["ip", "link", "del", tap], check=False)
405
+
subprocess.run(["ip", "link", "del", bridge], check=False)
406
+
407
+
log(f"artifacts kept in {workdir}")
408
+
409
+
410
+
if __name__ == "__main__":
411
+
sys.exit(main())