My nix-darwin and NixOS config
feat: replace Cockpit with Grafana+Prometheus for server monitoring
Remove Cockpit service, Caddy vhosts, split-dns entry, and options.
Add Grafana (tailnet-only) with Prometheus scraping node_exporter,
Caddy, Immich, and Postgres. Nextcloud exporter gated behind
nextcloudMetrics flag until token is provisioned.
+190
-59
+1
hosts/server/default.nix
+1
hosts/server/default.nix
+172
modules/grafana.nix
+172
modules/grafana.nix
···
1
+
##############################################################################
2
+
# Grafana + Prometheus — tailnet-only observability stack.
3
+
#
4
+
# Architecture:
5
+
# Prometheus scrapes:
6
+
# • node_exporter — system metrics (CPU, mem, disk, net)
7
+
# • caddy — request rates, latency, TLS
8
+
# • nextcloud — user counts, file counts, app status
9
+
# • immich — job queue, library stats
10
+
# • jellyfin — via prometheus-jellyfin-exporter
11
+
# • postgres (×2) — Nextcloud and Immich DB stats
12
+
# Grafana reads from Prometheus and serves dashboards.
13
+
#
14
+
# Access:
15
+
# https://grafana.ewancroft.uk — tailnet only, via Caddy on Tailscale IP.
16
+
# Prometheus itself is localhost-only; never exposed directly.
17
+
##############################################################################
18
+
{
19
+
config,
20
+
lib,
21
+
...
22
+
}:
23
+
let
24
+
cfg = config.myConfig;
25
+
hasTailnet = cfg.server.tailscaleIP != "";
26
+
gf = cfg.server.grafana;
27
+
28
+
prometheusPort = 9090;
29
+
grafanaPort = gf.port;
30
+
31
+
# Scrape configs built conditionally per enabled service
32
+
scrapeConfigs =
33
+
[
34
+
# ── System ──────────────────────────────────────────────────────────────
35
+
{
36
+
job_name = "node";
37
+
static_configs = [ { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ]; } ];
38
+
}
39
+
# ── Caddy ───────────────────────────────────────────────────────────────
40
+
{
41
+
job_name = "caddy";
42
+
static_configs = [ { targets = [ "127.0.0.1:2019" ]; } ];
43
+
metrics_path = "/metrics";
44
+
}
45
+
]
46
+
++ lib.optional (cfg.services.nextcloud.enable && cfg.server.grafana.nextcloudMetrics) {
47
+
job_name = "nextcloud";
48
+
static_configs = [ { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.nextcloud.port}" ]; } ];
49
+
}
50
+
++ lib.optional cfg.services.immich.enable {
51
+
job_name = "immich";
52
+
static_configs = [
53
+
# Immich exposes two metrics endpoints on separate ports
54
+
{ targets = [ "127.0.0.1:8081" "127.0.0.1:8082" ]; }
55
+
];
56
+
}
57
+
++ lib.optional cfg.services.postgresql.enable {
58
+
job_name = "postgres";
59
+
static_configs = [ { targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.postgres.port}" ]; } ];
60
+
};
61
+
in
62
+
lib.mkIf hasTailnet {
63
+
64
+
# ── Prometheus exporters ─────────────────────────────────────────────────
65
+
services.prometheus.exporters = {
66
+
node = {
67
+
enable = true;
68
+
port = 9100;
69
+
enabledCollectors = [
70
+
"cpu"
71
+
"diskstats"
72
+
"filesystem"
73
+
"loadavg"
74
+
"meminfo"
75
+
"netdev"
76
+
"stat"
77
+
"systemd"
78
+
"time"
79
+
"uname"
80
+
];
81
+
};
82
+
83
+
# Nextcloud exporter is disabled until the Monitoring app is installed
84
+
# and secrets/nextcloud-metrics-token is created. Enable by setting
85
+
# myConfig.server.grafana.nextcloudMetrics = true in the host config.
86
+
nextcloud = lib.mkIf (cfg.services.nextcloud.enable && cfg.server.grafana.nextcloudMetrics) {
87
+
enable = true;
88
+
port = 9205;
89
+
url = "https://${cfg.nextcloud.hostname}";
90
+
tokenFile = config.sops.secrets."nextcloud-metrics-token".path;
91
+
};
92
+
93
+
postgres = lib.mkIf config.services.postgresql.enable {
94
+
enable = true;
95
+
port = 9187;
96
+
runAsLocalSuperUser = true;
97
+
};
98
+
};
99
+
100
+
# Nextcloud metrics token — generate in Nextcloud admin → Monitoring app,
101
+
# then: sops secrets/nextcloud-metrics-token (binary, raw token).
102
+
# Only activated when myConfig.server.grafana.nextcloudMetrics = true.
103
+
sops.secrets."nextcloud-metrics-token" = lib.mkIf (cfg.services.nextcloud.enable && cfg.server.grafana.nextcloudMetrics) {
104
+
sopsFile = ../secrets/nextcloud-metrics-token;
105
+
format = "binary";
106
+
owner = "nextcloud-exporter";
107
+
mode = "0440";
108
+
};
109
+
110
+
# ── Immich metrics ───────────────────────────────────────────────────────
111
+
# Enable Immich's built-in Prometheus endpoints (server: 8081, microservices: 8082).
112
+
services.immich.metrics.enable = lib.mkIf cfg.services.immich.enable true;
113
+
114
+
# ── Caddy metrics ────────────────────────────────────────────────────────
115
+
# Expose /metrics on the admin API port (2019, localhost only by default).
116
+
services.caddy.globalConfig = lib.mkAfter ''
117
+
servers {
118
+
metrics
119
+
}
120
+
'';
121
+
122
+
# ── Prometheus ───────────────────────────────────────────────────────────
123
+
services.prometheus = {
124
+
enable = true;
125
+
port = prometheusPort;
126
+
listenAddress = "127.0.0.1";
127
+
retentionTime = "30d";
128
+
inherit scrapeConfigs;
129
+
};
130
+
131
+
# ── Grafana ──────────────────────────────────────────────────────────────
132
+
services.grafana = {
133
+
enable = true;
134
+
settings = {
135
+
server = {
136
+
http_addr = "127.0.0.1";
137
+
http_port = grafanaPort;
138
+
domain = gf.hostname;
139
+
root_url = "https://${gf.hostname}";
140
+
enforce_domain = false;
141
+
};
142
+
analytics.reporting_enabled = false;
143
+
};
144
+
provision = {
145
+
enable = true;
146
+
datasources.settings.datasources = [
147
+
{
148
+
name = "Prometheus";
149
+
type = "prometheus";
150
+
url = "http://127.0.0.1:${toString prometheusPort}";
151
+
isDefault = true;
152
+
}
153
+
];
154
+
};
155
+
};
156
+
157
+
# ── Caddy vhosts ─────────────────────────────────────────────────────────
158
+
services.caddy.virtualHosts."http://${gf.hostname}" = {
159
+
extraConfig = ''
160
+
bind ${cfg.server.tailscaleIP}
161
+
redir https://${gf.hostname}{uri} permanent
162
+
'';
163
+
};
164
+
165
+
services.caddy.virtualHosts."https://${gf.hostname}" = {
166
+
extraConfig = ''
167
+
bind ${cfg.server.tailscaleIP}
168
+
tls ${cfg.server.acmeCertDir}/fullchain.pem ${cfg.server.acmeCertDir}/key.pem
169
+
reverse_proxy http://127.0.0.1:${toString grafanaPort}
170
+
'';
171
+
};
172
+
}
+16
-14
modules/options.nix
+16
-14
modules/options.nix
···
746
746
};
747
747
};
748
748
749
-
acmeCertDir = mkOption {
750
-
type = str;
751
-
default = "/var/lib/acme/ewancroft.uk";
752
-
description = "Directory containing the ACME wildcard cert for *.ewancroft.uk, used by Caddy tailnet vhosts.";
753
-
};
754
-
755
-
cockpit = {
756
-
enable = mkOption {
757
-
type = bool;
758
-
default = true;
759
-
};
749
+
grafana = {
760
750
hostname = mkOption {
761
751
type = str;
762
-
default = "cockpit.ewancroft.uk";
763
-
description = "Hostname for the Cockpit tailnet vhost — resolved to the server's Tailscale IP via split-dns.";
752
+
default = "grafana.ewancroft.uk";
753
+
description = "Hostname for the Grafana dashboard (tailnet only).";
764
754
};
765
755
port = mkOption {
766
756
type = int;
767
-
default = 9090;
757
+
default = 3000;
758
+
description = "Local port Grafana listens on.";
759
+
};
760
+
nextcloudMetrics = mkOption {
761
+
type = bool;
762
+
default = false;
763
+
description = "Enable Nextcloud exporter. Requires the Monitoring app and secrets/nextcloud-metrics-token.";
768
764
};
765
+
};
766
+
767
+
acmeCertDir = mkOption {
768
+
type = str;
769
+
default = "/var/lib/acme/ewancroft.uk";
770
+
description = "Directory containing the ACME wildcard cert for *.ewancroft.uk, used by Caddy tailnet vhosts.";
769
771
};
770
772
771
773
tailscaleIP = mkOption {
-44
modules/server/services.nix
-44
modules/server/services.nix
···
5
5
}:
6
6
let
7
7
cfg = config.myConfig;
8
-
cockpit = cfg.server.cockpit;
9
8
in
10
9
{
11
10
# Tailscale VPN for inter-host communication
12
11
services.tailscale.enable = true;
13
12
14
-
# PCP (Performance Co-Pilot) — enables historical graphs in Cockpit.
15
-
services.pcp.enable = true;
16
-
17
13
# SSH daemon (server hardened configuration from modules/server/ssh.nix)
18
14
# No additional SSH config needed here — it's handled by server-hardened profile.
19
-
20
-
# ── Cockpit — tailnet-only web management console ──────────────────────────
21
-
# Cockpit listens on localhost only; Caddy proxies it on the Tailscale IP.
22
-
# Reachable at https://${cockpit.hostname} from any tailnet device once
23
-
# split-dns is configured (see modules/split-dns.nix).
24
-
services.cockpit = lib.mkIf cockpit.enable {
25
-
enable = true;
26
-
port = cockpit.port;
27
-
settings.WebService = {
28
-
# Required when behind a reverse proxy — Cockpit rejects WebSocket
29
-
# connections from origins it doesn't recognise.
30
-
Origins = lib.mkForce "https://${cockpit.hostname} wss://${cockpit.hostname}";
31
-
# Tell Cockpit to trust the X-Forwarded-Proto header from Caddy.
32
-
ProtocolHeader = "X-Forwarded-Proto";
33
-
};
34
-
};
35
-
36
-
services.caddy.virtualHosts."http://${cockpit.hostname}" =
37
-
lib.mkIf (cockpit.enable && cfg.server.tailscaleIP != "")
38
-
{
39
-
extraConfig = ''
40
-
bind ${cfg.server.tailscaleIP}
41
-
redir https://${cockpit.hostname}{uri} permanent
42
-
'';
43
-
};
44
-
45
-
services.caddy.virtualHosts."https://${cockpit.hostname}" =
46
-
lib.mkIf (cockpit.enable && cfg.server.tailscaleIP != "")
47
-
{
48
-
extraConfig = ''
49
-
bind ${cfg.server.tailscaleIP}
50
-
tls ${cfg.server.acmeCertDir}/fullchain.pem ${cfg.server.acmeCertDir}/key.pem
51
-
reverse_proxy http://127.0.0.1:${toString cockpit.port} {
52
-
header_up X-Forwarded-Proto https
53
-
transport http {
54
-
read_timeout 30s
55
-
}
56
-
}
57
-
'';
58
-
};
59
15
}
+1
-1
modules/split-dns.nix
+1
-1
modules/split-dns.nix
···
37
37
lib.optional cfg.services.immich.enable "${tsIP} ${cfg.immich.hostname}"
38
38
++ lib.optional cfg.services.nextcloud.enable "${tsIP} ${cfg.nextcloud.hostname}"
39
39
++ lib.optional cfg.services.jellyfin.enable "${tsIP} ${cfg.jellyfin.hostname}"
40
-
++ lib.optional cfg.server.cockpit.enable "${tsIP} ${cfg.server.cockpit.hostname}"
40
+
++ [ "${tsIP} ${cfg.server.grafana.hostname}" ]
41
41
);
42
42
in
43
43
lib.mkIf (tsIP != "") {