this repo has no description
4
fork

Configure Feed

Select the types of activity you want to include in your feed.

feat(wolumonde): setup the logging stuff, actually works now

dusk d4a0f31e 10c55292

+174 -75
+15 -29
hosts/wolumonde/modules/fluentbit.nix
··· 8 8 services.fluent-bit = { 9 9 enable = true; 10 10 settings = { 11 - parsers = [ 11 + service.flush = 1; 12 + pipeline.inputs = [ 12 13 { 13 - name = "nginx"; 14 - format = "regex"; 15 - regex = ''^(?<remote_addr>[^ ]+) - (?<remote_user>[^ ]+) \[(?<time_local>[^\]]+)\] "(?<request>[^"]*)" (?<status>\d{3}) (?<body_bytes_sent>\d+) "(?<http_referer>[^"]*)" "(?<http_user_agent>[^"]*)" (?<request_time>[0-9\.]+)$''; 16 - time_key = "time_local"; 17 - time_format = "%d/%b/%Y:%H:%M:%S %z"; 18 - time_keep = "off"; 14 + name = "node_exporter_metrics"; 15 + tag = "metrics.node"; 16 + scrape_interval = 5; 17 + } 18 + # { 19 + # name = "dummy"; 20 + # tag = "logs.dummy"; 21 + # dummy = ''{"_msg": "dummy"}''; 22 + # } 23 + { 24 + name = "fluentbit_metrics"; 25 + tag = "metrics.fluentbit"; 26 + scrape_interval = 5; 19 27 } 20 28 ]; 21 - pipeline = { 22 - inputs = [ 23 - { 24 - name = "tail"; 25 - tag = "nginx.access"; 26 - path = "/var/lib/nginx/access.log"; 27 - db = "/var/lib/fluent-bit/nginx-access.db"; 28 - parser = "nginx"; 29 - } 30 - ]; 31 - outputs = [ 32 - { 33 - name = "http"; 34 - match = "nginx.access"; 35 - host = "127.0.0.1"; 36 - port = lib.removePrefix ":" config.services.victorialogs.listenAddress; 37 - uri = "/insert/jsonline?_stream_fields=stream&_msg_field=log&_time_field=date"; 38 - format = "json_lines"; 39 - json_date_format = "iso8601"; 40 - } 41 - ]; 42 - }; 43 29 }; 44 30 }; 45 31
+65 -6
hosts/wolumonde/modules/nginx.nix
··· 1 - { inputs, pkgs, ... }: 1 + { 2 + config, 3 + lib, 4 + inputs, 5 + pkgs, 6 + ... 7 + }: 2 8 { 3 9 services.nginx = { 4 10 enable = true; ··· 7 13 recommendedOptimisation = true; 8 14 recommendedGzipSettings = true; 9 15 recommendedProxySettings = true; 16 + # /nginx_status 10 17 statusPage = true; 11 18 }; 12 19 20 + # output json logs so we can consume them more easily 21 + services.nginx.appendHttpConfig = '' 22 + log_format json_logs escape=json '{' 23 + '"_msg":"request completed",' 24 + '"time":"$time_local",' 25 + '"req.remoteAddr":"$remote_addr",' 26 + '"req.method":"$request_method",' 27 + '"req.url":"$uri",' 28 + '"req.httpVersion":"$server_protocol",' 29 + '"res.statusCode":$status,' 30 + '"res.bodySize":$body_bytes_sent,' 31 + '"req.headers.id":"$request_id",' 32 + '"req.headers.referer":"$http_referer",' 33 + '"req.headers.user-agent":"$http_user_agent",' 34 + '"responseTime":$request_time' 35 + '}'; 36 + access_log /var/log/nginx/access.log json_logs; 37 + ''; 38 + 13 39 users.users.nginx.extraGroups = [ "acme" ]; 14 40 15 41 security.acme = { ··· 35 61 }; 36 62 }; 37 63 38 - services.prometheus.exporters.nginx = { 39 - enable = true; 40 - port = 9113; 64 + services.fluent-bit.settings = { 65 + parsers = [ 66 + { 67 + name = "nginx_json"; 68 + format = "json"; 69 + time_key = "time"; 70 + time_format = "%d/%b/%Y:%H:%M:%S %z"; 71 + } 72 + ]; 73 + pipeline = { 74 + inputs = [ 75 + { 76 + name = "nginx_metrics"; 77 + tag = "metrics.nginx"; 78 + status_url = "/nginx_status"; 79 + nginx_plus = false; 80 + } 81 + { 82 + name = "tail"; 83 + tag = "logs.nginx"; 84 + path = "/var/log/nginx/*.log"; 85 + db = "/var/lib/fluent-bit/nginx-access.db"; 86 + "db.locking" = true; 87 + buffer_chunk_size = "4m"; 88 + buffer_max_size = "32m"; 89 + parser = "nginx_json"; 90 + } 91 + ]; 92 + }; 41 93 }; 94 + 95 + # need so fluent-bit can access nginx 96 + systemd.services.fluent-bit.serviceConfig.SupplementaryGroups = lib.mkForce "systemd-journal nginx"; 42 97 43 98 services.vmalert.rules.groups = [ 44 99 { ··· 51 106 expr = "* | stats count() as requests"; 52 107 } 53 108 { 109 + record = "nginx_2xx_count"; 110 + expr = ''* | res.statusCode:~"2.." | stats count() as successes''; 111 + } 112 + { 54 113 record = "nginx_5xx_count"; 55 - expr = ''* | status:~"5.." | stats count() as errors''; 114 + expr = ''* | res.statusCode:~"5.." | stats count() as errors''; 56 115 } 57 116 { 58 117 record = "nginx_request_latency_avg"; 59 - expr = "* | stats avg(request_time) as avg_latency"; 118 + expr = "* | stats avg(responseTime) as avg_latency"; 60 119 } 61 120 ]; 62 121 }
-6
hosts/wolumonde/modules/node-exporter.nix
··· 1 - { 2 - services.prometheus.exporters.node = { 3 - enable = true; 4 - port = 9100; # default 5 - }; 6 - }
+34
hosts/wolumonde/modules/pds.nix
··· 11 11 extraConfig = '' 12 12 proxy_set_header Upgrade $http_upgrade; 13 13 proxy_set_header Connection $connection_upgrade; 14 + proxy_set_header id $request_id; 14 15 ''; 15 16 # higher prio just to make sure 16 17 priority = 100; ··· 37 38 PDS_CRAWLERS = "https://bsky.network"; 38 39 }; 39 40 environmentFiles = [ config.age.secrets.pdsConfig.path ]; 41 + }; 42 + 43 + services.fluent-bit.settings = { 44 + parsers = [ 45 + { 46 + name = "pds_json"; 47 + format = "json"; 48 + time_key = "time"; 49 + time_strict = false; 50 + } 51 + ]; 52 + pipeline = { 53 + inputs = [ 54 + { 55 + name = "systemd"; 56 + tag = "logs.pds"; 57 + systemd_filter = "_SYSTEMD_UNIT=pds.service"; 58 + } 59 + ]; 60 + filters = [ 61 + { 62 + name = "parser"; 63 + match = "logs.pds"; 64 + key_name = "MESSAGE"; 65 + parser = "pds_json"; 66 + } 67 + { 68 + name = "modify"; 69 + match = "logs.pds"; 70 + Rename = [ "msg _msg" ]; 71 + } 72 + ]; 73 + }; 40 74 }; 41 75 42 76 # virtualisation = {
+5 -2
hosts/wolumonde/modules/perses.nix
··· 18 18 config.Cmd = [ 19 19 "--config=/etc/perses/config.yaml" 20 20 "--log.level=info" 21 + "--web.listen-address=:${toString port}" 21 22 # "--log.method-trace" 22 23 ]; 23 24 config.Healthcheck = { 24 25 Test = [ 25 26 "/bin/curl" 26 - "http://localhost:8080/api/v1/health" 27 + "http://localhost:${toString port}/api/v1/health" 27 28 ]; 28 29 Retries = 3; 29 30 }; ··· 89 90 volumes = [ 90 91 "/var/lib/perses:/perses" 91 92 ]; 92 - ports = [ "${toString port}:8080" ]; 93 + extraOptions = [ 94 + "--network=host" 95 + ]; 93 96 }; 94 97 95 98 services.nginx.virtualHosts.${domain} = {
+55 -32
hosts/wolumonde/modules/victoria.nix
··· 1 - { config, ... }: 1 + { lib, config, ... }: 2 + let 3 + syslogUdp = 5113; 4 + in 2 5 { 3 - # Enable single-node VictoriaMetrics on port 8428 (default) 4 6 services.victoriametrics = { 5 7 enable = true; 6 - listenAddress = ":8428"; # default port for metrics 7 - prometheusConfig = { 8 - scrape_configs = [ 9 - { 10 - job_name = "node"; 11 - static_configs = [ 12 - { 13 - targets = [ "localhost:9100" ]; 14 - labels.type = "node"; 15 - } 16 - ]; 17 - } 18 - { 19 - job_name = "nginx"; 20 - static_configs = [ { targets = [ "localhost:9113" ]; } ]; 21 - } 22 - ]; 23 - }; 8 + listenAddress = ":8428"; 24 9 }; 25 10 26 - # Enable VictoriaLogs (logs database) on port 9428 (default) 27 11 services.victorialogs = { 28 12 enable = true; 29 - listenAddress = ":9428"; # default port for logs 30 - # You can add extra options if needed, e.g. authentication or retention 31 - # extraOptions = [ "-loggerLevel=INFO" ]; 13 + listenAddress = ":9428"; 14 + # extraOptions = ["-syslog.listenAddr.udp=:${toString syslogUdp}" "-journald.maxRequestSize=1024000000"]; 32 15 }; 33 16 34 - # Enable vmalert for LogsQL recording rules 35 17 services.vmalert = { 36 18 enable = true; 37 - # Point vmalert to VictoriaLogs and VictoriaMetrics 38 - settings = { 39 - "datasource.url" = "http://127.0.0.1${config.services.victorialogs.listenAddress}"; # VictoriaLogs address 40 - "remoteWrite.url" = "http://127.0.0.1${config.services.victoriametrics.listenAddress}"; # Remote-write to VictoriaMetrics 41 - "remoteRead.url" = "http://127.0.0.1${config.services.victoriametrics.listenAddress}"; # Remote-read from VictoriaMetrics 42 - "rule.defaultRuleType" = "vlogs"; # Use LogsQL rules by default 43 - }; 19 + settings = 20 + let 21 + l = "http://localhost"; 22 + in 23 + { 24 + "datasource.url" = "${l}${config.services.victorialogs.listenAddress}"; 25 + "remoteWrite.url" = "${l}${config.services.victoriametrics.listenAddress}"; 26 + "remoteRead.url" = "${l}${config.services.victoriametrics.listenAddress}"; 27 + "rule.defaultRuleType" = "vlogs"; 28 + }; 44 29 }; 30 + 31 + services.fluent-bit.settings.pipeline.outputs = [ 32 + # write metrics to victoriametrics via prometheus 33 + { 34 + name = "prometheus_remote_write"; 35 + match = "metrics.*"; 36 + port = lib.removePrefix ":" config.services.victoriametrics.listenAddress; 37 + uri = "/api/v1/write"; 38 + } 39 + { 40 + name = "http"; 41 + match = "logs.*"; 42 + port = lib.removePrefix ":" config.services.victorialogs.listenAddress; 43 + uri = "/insert/jsonline?_stream_fields=stream&_msg_field=log&_time_field=date"; 44 + format = "json_lines"; 45 + json_date_format = "iso8601"; 46 + } 47 + # write logs via syslog 48 + # { 49 + # name = "syslog"; 50 + # match = "*.log"; 51 + # port = syslogUdp; 52 + # syslog_maxsize = 4096; 53 + # syslog_severity_key = "severity"; 54 + # syslog_facility_key = "facility"; 55 + # syslog_hostname_key = "hostname"; 56 + # syslog_appname_key = "appname"; 57 + # syslog_procid_key = "procid"; 58 + # syslog_msgid_key = "msgid"; 59 + # syslog_sd_key = "sd"; 60 + # syslog_message_key = "message"; 61 + # } 62 + ]; 63 + 64 + # services.journald.upload = { 65 + # enable = true; 66 + # settings.Upload.URL = "http://localhost${config.services.victorialogs.listenAddress}/insert/journald"; 67 + # }; 45 68 }