this repo has no description
feat(wolumonde): setup the logging stuff, actually works now
+174
-75
+15
-29
hosts/wolumonde/modules/fluentbit.nix
+15
-29
hosts/wolumonde/modules/fluentbit.nix
···
8
8
services.fluent-bit = {
9
9
enable = true;
10
10
settings = {
11
-
parsers = [
11
+
service.flush = 1;
12
+
pipeline.inputs = [
12
13
{
13
-
name = "nginx";
14
-
format = "regex";
15
-
regex = ''^(?<remote_addr>[^ ]+) - (?<remote_user>[^ ]+) \[(?<time_local>[^\]]+)\] "(?<request>[^"]*)" (?<status>\d{3}) (?<body_bytes_sent>\d+) "(?<http_referer>[^"]*)" "(?<http_user_agent>[^"]*)" (?<request_time>[0-9\.]+)$'';
16
-
time_key = "time_local";
17
-
time_format = "%d/%b/%Y:%H:%M:%S %z";
18
-
time_keep = "off";
14
+
name = "node_exporter_metrics";
15
+
tag = "metrics.node";
16
+
scrape_interval = 5;
17
+
}
18
+
# {
19
+
# name = "dummy";
20
+
# tag = "logs.dummy";
21
+
# dummy = ''{"_msg": "dummy"}'';
22
+
# }
23
+
{
24
+
name = "fluentbit_metrics";
25
+
tag = "metrics.fluentbit";
26
+
scrape_interval = 5;
19
27
}
20
28
];
21
-
pipeline = {
22
-
inputs = [
23
-
{
24
-
name = "tail";
25
-
tag = "nginx.access";
26
-
path = "/var/lib/nginx/access.log";
27
-
db = "/var/lib/fluent-bit/nginx-access.db";
28
-
parser = "nginx";
29
-
}
30
-
];
31
-
outputs = [
32
-
{
33
-
name = "http";
34
-
match = "nginx.access";
35
-
host = "127.0.0.1";
36
-
port = lib.removePrefix ":" config.services.victorialogs.listenAddress;
37
-
uri = "/insert/jsonline?_stream_fields=stream&_msg_field=log&_time_field=date";
38
-
format = "json_lines";
39
-
json_date_format = "iso8601";
40
-
}
41
-
];
42
-
};
43
29
};
44
30
};
45
31
+65
-6
hosts/wolumonde/modules/nginx.nix
+65
-6
hosts/wolumonde/modules/nginx.nix
···
1
-
{ inputs, pkgs, ... }:
1
+
{
2
+
config,
3
+
lib,
4
+
inputs,
5
+
pkgs,
6
+
...
7
+
}:
2
8
{
3
9
services.nginx = {
4
10
enable = true;
···
7
13
recommendedOptimisation = true;
8
14
recommendedGzipSettings = true;
9
15
recommendedProxySettings = true;
16
+
# /nginx_status
10
17
statusPage = true;
11
18
};
12
19
20
+
# output json logs so we can consume them more easily
21
+
services.nginx.appendHttpConfig = ''
22
+
log_format json_logs escape=json '{'
23
+
'"_msg":"request completed",'
24
+
'"time":"$time_local",'
25
+
'"req.remoteAddr":"$remote_addr",'
26
+
'"req.method":"$request_method",'
27
+
'"req.url":"$uri",'
28
+
'"req.httpVersion":"$server_protocol",'
29
+
'"res.statusCode":$status,'
30
+
'"res.bodySize":$body_bytes_sent,'
31
+
'"req.headers.id":"$request_id",'
32
+
'"req.headers.referer":"$http_referer",'
33
+
'"req.headers.user-agent":"$http_user_agent",'
34
+
'"responseTime":$request_time'
35
+
'}';
36
+
access_log /var/log/nginx/access.log json_logs;
37
+
'';
38
+
13
39
users.users.nginx.extraGroups = [ "acme" ];
14
40
15
41
security.acme = {
···
35
61
};
36
62
};
37
63
38
-
services.prometheus.exporters.nginx = {
39
-
enable = true;
40
-
port = 9113;
64
+
services.fluent-bit.settings = {
65
+
parsers = [
66
+
{
67
+
name = "nginx_json";
68
+
format = "json";
69
+
time_key = "time";
70
+
time_format = "%d/%b/%Y:%H:%M:%S %z";
71
+
}
72
+
];
73
+
pipeline = {
74
+
inputs = [
75
+
{
76
+
name = "nginx_metrics";
77
+
tag = "metrics.nginx";
78
+
status_url = "/nginx_status";
79
+
nginx_plus = false;
80
+
}
81
+
{
82
+
name = "tail";
83
+
tag = "logs.nginx";
84
+
path = "/var/log/nginx/*.log";
85
+
db = "/var/lib/fluent-bit/nginx-access.db";
86
+
"db.locking" = true;
87
+
buffer_chunk_size = "4m";
88
+
buffer_max_size = "32m";
89
+
parser = "nginx_json";
90
+
}
91
+
];
92
+
};
41
93
};
94
+
95
+
# need so fluent-bit can access nginx
96
+
systemd.services.fluent-bit.serviceConfig.SupplementaryGroups = lib.mkForce "systemd-journal nginx";
42
97
43
98
services.vmalert.rules.groups = [
44
99
{
···
51
106
expr = "* | stats count() as requests";
52
107
}
53
108
{
109
+
record = "nginx_2xx_count";
110
+
expr = ''* | res.statusCode:~"2.." | stats count() as successes'';
111
+
}
112
+
{
54
113
record = "nginx_5xx_count";
55
-
expr = ''* | status:~"5.." | stats count() as errors'';
114
+
expr = ''* | res.statusCode:~"5.." | stats count() as errors'';
56
115
}
57
116
{
58
117
record = "nginx_request_latency_avg";
59
-
expr = "* | stats avg(request_time) as avg_latency";
118
+
expr = "* | stats avg(responseTime) as avg_latency";
60
119
}
61
120
];
62
121
}
-6
hosts/wolumonde/modules/node-exporter.nix
-6
hosts/wolumonde/modules/node-exporter.nix
+34
hosts/wolumonde/modules/pds.nix
+34
hosts/wolumonde/modules/pds.nix
···
11
11
extraConfig = ''
12
12
proxy_set_header Upgrade $http_upgrade;
13
13
proxy_set_header Connection $connection_upgrade;
14
+
proxy_set_header id $request_id;
14
15
'';
15
16
# higher prio just to make sure
16
17
priority = 100;
···
37
38
PDS_CRAWLERS = "https://bsky.network";
38
39
};
39
40
environmentFiles = [ config.age.secrets.pdsConfig.path ];
41
+
};
42
+
43
+
services.fluent-bit.settings = {
44
+
parsers = [
45
+
{
46
+
name = "pds_json";
47
+
format = "json";
48
+
time_key = "time";
49
+
time_strict = false;
50
+
}
51
+
];
52
+
pipeline = {
53
+
inputs = [
54
+
{
55
+
name = "systemd";
56
+
tag = "logs.pds";
57
+
systemd_filter = "_SYSTEMD_UNIT=pds.service";
58
+
}
59
+
];
60
+
filters = [
61
+
{
62
+
name = "parser";
63
+
match = "logs.pds";
64
+
key_name = "MESSAGE";
65
+
parser = "pds_json";
66
+
}
67
+
{
68
+
name = "modify";
69
+
match = "logs.pds";
70
+
Rename = [ "msg _msg" ];
71
+
}
72
+
];
73
+
};
40
74
};
41
75
42
76
# virtualisation = {
+5
-2
hosts/wolumonde/modules/perses.nix
+5
-2
hosts/wolumonde/modules/perses.nix
···
18
18
config.Cmd = [
19
19
"--config=/etc/perses/config.yaml"
20
20
"--log.level=info"
21
+
"--web.listen-address=:${toString port}"
21
22
# "--log.method-trace"
22
23
];
23
24
config.Healthcheck = {
24
25
Test = [
25
26
"/bin/curl"
26
-
"http://localhost:8080/api/v1/health"
27
+
"http://localhost:${toString port}/api/v1/health"
27
28
];
28
29
Retries = 3;
29
30
};
···
89
90
volumes = [
90
91
"/var/lib/perses:/perses"
91
92
];
92
-
ports = [ "${toString port}:8080" ];
93
+
extraOptions = [
94
+
"--network=host"
95
+
];
93
96
};
94
97
95
98
services.nginx.virtualHosts.${domain} = {
+55
-32
hosts/wolumonde/modules/victoria.nix
+55
-32
hosts/wolumonde/modules/victoria.nix
···
1
-
{ config, ... }:
1
+
{ lib, config, ... }:
2
+
let
3
+
syslogUdp = 5113;
4
+
in
2
5
{
3
-
# Enable single-node VictoriaMetrics on port 8428 (default)
4
6
services.victoriametrics = {
5
7
enable = true;
6
-
listenAddress = ":8428"; # default port for metrics
7
-
prometheusConfig = {
8
-
scrape_configs = [
9
-
{
10
-
job_name = "node";
11
-
static_configs = [
12
-
{
13
-
targets = [ "localhost:9100" ];
14
-
labels.type = "node";
15
-
}
16
-
];
17
-
}
18
-
{
19
-
job_name = "nginx";
20
-
static_configs = [ { targets = [ "localhost:9113" ]; } ];
21
-
}
22
-
];
23
-
};
8
+
listenAddress = ":8428";
24
9
};
25
10
26
-
# Enable VictoriaLogs (logs database) on port 9428 (default)
27
11
services.victorialogs = {
28
12
enable = true;
29
-
listenAddress = ":9428"; # default port for logs
30
-
# You can add extra options if needed, e.g. authentication or retention
31
-
# extraOptions = [ "-loggerLevel=INFO" ];
13
+
listenAddress = ":9428";
14
+
# extraOptions = ["-syslog.listenAddr.udp=:${toString syslogUdp}" "-journald.maxRequestSize=1024000000"];
32
15
};
33
16
34
-
# Enable vmalert for LogsQL recording rules
35
17
services.vmalert = {
36
18
enable = true;
37
-
# Point vmalert to VictoriaLogs and VictoriaMetrics
38
-
settings = {
39
-
"datasource.url" = "http://127.0.0.1${config.services.victorialogs.listenAddress}"; # VictoriaLogs address
40
-
"remoteWrite.url" = "http://127.0.0.1${config.services.victoriametrics.listenAddress}"; # Remote-write to VictoriaMetrics
41
-
"remoteRead.url" = "http://127.0.0.1${config.services.victoriametrics.listenAddress}"; # Remote-read from VictoriaMetrics
42
-
"rule.defaultRuleType" = "vlogs"; # Use LogsQL rules by default
43
-
};
19
+
settings =
20
+
let
21
+
l = "http://localhost";
22
+
in
23
+
{
24
+
"datasource.url" = "${l}${config.services.victorialogs.listenAddress}";
25
+
"remoteWrite.url" = "${l}${config.services.victoriametrics.listenAddress}";
26
+
"remoteRead.url" = "${l}${config.services.victoriametrics.listenAddress}";
27
+
"rule.defaultRuleType" = "vlogs";
28
+
};
44
29
};
30
+
31
+
services.fluent-bit.settings.pipeline.outputs = [
32
+
# write metrics to victoriametrics via prometheus
33
+
{
34
+
name = "prometheus_remote_write";
35
+
match = "metrics.*";
36
+
port = lib.removePrefix ":" config.services.victoriametrics.listenAddress;
37
+
uri = "/api/v1/write";
38
+
}
39
+
{
40
+
name = "http";
41
+
match = "logs.*";
42
+
port = lib.removePrefix ":" config.services.victorialogs.listenAddress;
43
+
uri = "/insert/jsonline?_stream_fields=stream&_msg_field=log&_time_field=date";
44
+
format = "json_lines";
45
+
json_date_format = "iso8601";
46
+
}
47
+
# write logs via syslog
48
+
# {
49
+
# name = "syslog";
50
+
# match = "*.log";
51
+
# port = syslogUdp;
52
+
# syslog_maxsize = 4096;
53
+
# syslog_severity_key = "severity";
54
+
# syslog_facility_key = "facility";
55
+
# syslog_hostname_key = "hostname";
56
+
# syslog_appname_key = "appname";
57
+
# syslog_procid_key = "procid";
58
+
# syslog_msgid_key = "msgid";
59
+
# syslog_sd_key = "sd";
60
+
# syslog_message_key = "message";
61
+
# }
62
+
];
63
+
64
+
# services.journald.upload = {
65
+
# enable = true;
66
+
# settings.Upload.URL = "http://localhost${config.services.victorialogs.listenAddress}/insert/journald";
67
+
# };
45
68
}