feat(nixos): k3s cluster with shared token · khuedoan.com/cloudlab@744778d

+7

infra/_modules/nixos/.sops.yaml

··· 1 + creation_rules: 2 + # TODO auto manage age keys, currently have to: 3 + # - run age-keygen 4 + # - copy public key here 5 + # - copy private key to ~/.config/sops/age/keys.txt 6 + - age: >- 7 + age15c5rpksj0u27sp667525zqh8dhtd70rwrlv7xq2hqhsfaempcdwqu3unel

+10

infra/_modules/nixos/configuration.nix

··· 63 63 }; 64 64 }; 65 65 66 + sops = { 67 + age = { 68 + keyFile = "/var/lib/secrets/age"; 69 + }; 70 + defaultSopsFile = ./secrets.yaml; 71 + secrets = { 72 + k3s_token = {}; 73 + }; 74 + }; 75 + 66 76 system.stateVersion = "25.05"; 67 77 }

+12

infra/_modules/nixos/decrypt-age-keys.sh

··· 1 + #!/usr/bin/env sh 2 + 3 + set -euo pipefail -x 4 + 5 + mkdir -p ./var/lib/secrets 6 + 7 + umask 0177 8 + sops \ 9 + --extract '["age_key"]' \ 10 + --decrypt "${SOPS_FILE}" \ 11 + > ./var/lib/secrets/age 12 + umask 0022

+22 -1

infra/_modules/nixos/flake.lock

··· 38 38 "root": { 39 39 "inputs": { 40 40 "disko": "disko", 41 - "nixpkgs": "nixpkgs" 41 + "nixpkgs": "nixpkgs", 42 + "sops-nix": "sops-nix" 43 + } 44 + }, 45 + "sops-nix": { 46 + "inputs": { 47 + "nixpkgs": [ 48 + "nixpkgs" 49 + ] 50 + }, 51 + "locked": { 52 + "lastModified": 1754988908, 53 + "narHash": "sha256-t+voe2961vCgrzPFtZxha0/kmFSHFobzF00sT8p9h0U=", 54 + "owner": "Mic92", 55 + "repo": "sops-nix", 56 + "rev": "3223c7a92724b5d804e9988c6b447a0d09017d48", 57 + "type": "github" 58 + }, 59 + "original": { 60 + "owner": "Mic92", 61 + "repo": "sops-nix", 62 + "type": "github" 42 63 } 43 64 } 44 65 },

+10 -3

infra/_modules/nixos/flake.nix

··· 5 5 url = "github:nix-community/disko"; 6 6 inputs.nixpkgs.follows = "nixpkgs"; 7 7 }; 8 + sops-nix = { 9 + url = "github:Mic92/sops-nix"; 10 + inputs.nixpkgs.follows = "nixpkgs"; 11 + }; 8 12 }; 9 - outputs = { nixpkgs, disko, ... }: { 13 + outputs = { nixpkgs, disko, sops-nix, ... }: { 10 14 nixosConfigurations = 11 15 let 12 16 hosts = builtins.fromJSON (builtins.readFile ./hosts.json); ··· 24 28 system = "x86_64-linux"; 25 29 modules = [ 26 30 disko.nixosModules.disko 31 + sops-nix.nixosModules.sops 27 32 ./configuration.nix 28 33 ./disks.nix 29 34 ./profiles/k3s.nix ··· 37 42 system = "x86_64-linux"; 38 43 modules = [ 39 44 disko.nixosModules.disko 45 + sops-nix.nixosModules.sops 40 46 ./configuration.nix 41 47 ./disks.nix 42 48 ./profiles/k3s.nix 43 49 { 44 50 networking.hostName = "kube-2"; 45 - services.k3s.serverAddr = hosts.kube-1.ipv6_address; 51 + services.k3s.serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; 46 52 } 47 53 ]; 48 54 }; ··· 50 56 system = "x86_64-linux"; 51 57 modules = [ 52 58 disko.nixosModules.disko 59 + sops-nix.nixosModules.sops 53 60 ./configuration.nix 54 61 ./disks.nix 55 62 ./profiles/k3s.nix 56 63 { 57 64 networking.hostName = "kube-3"; 58 - services.k3s.serverAddr = hosts.kube-1.ipv6_address; 65 + services.k3s.serverAddr = "https://[${hosts.kube-1.ipv6_address}]:6443"; 59 66 } 60 67 ]; 61 68 };

+1 -1

infra/_modules/nixos/hosts.json

··· 1 - {"kube-1":{"ipv6_address":"2402:800:63e2:5af5:68d0:b3c2:b44:71cc"},"kube-2":{"ipv6_address":"2402:800:63e2:5af5:19a6:50a1:765a:e396"},"kube-3":{"ipv6_address":"2402:800:63e2:5af5:3646:e232:b1cd:c96a"}} 1 + {"kube-1":{"ipv6_address":"2402:800:63e2:5af5:1b0:f4a:e854:5355"},"kube-2":{"ipv6_address":"2402:800:63e2:5af5:cb46:f8e4:131b:93c5"},"kube-3":{"ipv6_address":"2402:800:63e2:5af5:1ced:4595:b4bf:3634"}}

+4

infra/_modules/nixos/main.tf

··· 17 17 nixos_partitioner_attr = "${var.flake}#nixosConfigurations.${each.key}.config.system.build.diskoScript" 18 18 target_host = each.value.ipv6_address 19 19 instance_id = each.key 20 + extra_files_script = "${path.module}/decrypt-age-keys.sh" 21 + extra_environment = { 22 + SOPS_FILE = var.sops_file 23 + } 20 24 }

+24

infra/_modules/nixos/profiles/installer.nix

··· 1 + { modulesPath, ... }: 2 + 3 + { 4 + imports = [ 5 + (modulesPath + "/installer/cd-dvd/installation-cd-minimal.nix") 6 + ]; 7 + 8 + services = { 9 + openssh = { 10 + enable = true; 11 + }; 12 + qemuGuest = { 13 + enable = true; 14 + }; 15 + }; 16 + 17 + users.users = { 18 + root = { 19 + openssh.authorizedKeys.keys = [ 20 + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN5ue4np7cF34f6dwqH1262fPjkowHQ8irfjVC156PCG" 21 + ]; 22 + }; 23 + }; 24 + }

+2 -1

infra/_modules/nixos/profiles/k3s.nix

··· 1 - { pkgs, ... }: 1 + { config, ... }: 2 2 3 3 { 4 4 networking = { ··· 18 18 k3s = { 19 19 enable = true; 20 20 role = "server"; 21 + tokenFile = config.sops.secrets.k3s_token.path; 21 22 extraFlags = toString [ 22 23 "--disable-helm-controller" 23 24 "--disable-network-policy"

+16

infra/_modules/nixos/secrets.yaml

··· 1 + k3s_token: ENC[AES256_GCM,data:Gkpz+zgUN4qHacbB52LOoVB4cm1OM7MrsEmKbqRvaukRnh4ctvBDd+9C+6fnOQsTNw==,iv:VD/YnqtVnu7PCGoGFCzzt8WrBAa6bdluNrRDO/YP2RA=,tag:+wu5MiCbMELVaIrEcK0KIQ==,type:str] 2 + sops: 3 + age: 4 + - recipient: age15c5rpksj0u27sp667525zqh8dhtd70rwrlv7xq2hqhsfaempcdwqu3unel 5 + enc: | 6 + -----BEGIN AGE ENCRYPTED FILE----- 7 + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSArSUFnczNDTzl1Mm8wVWdN 8 + NnZCN2hJa3JMOW42Mno0UGZod3hYMjhvWm5nCkpqRklwRSt6Sm9NWFA3aFEvYncv 9 + QjFqRng3TVhVRERDUkZwNTV2YThocTgKLS0tIGoyc0oxbU5FenhHVVJZTXpkSlhC 10 + eFpDWWg4Z2dWc3JvV2l0Q2Evd25mSTAKfZn8kohXG6O7y54dPpSXhRj1E75XGZDj 11 + CRxObtI2s+ddACiHzNVDuDJkq5RForvOYwR/6nk6jEdIQt8pfnwiSA== 12 + -----END AGE ENCRYPTED FILE----- 13 + lastmodified: "2025-08-27T13:36:25Z" 14 + mac: ENC[AES256_GCM,data:uEdpYAK1kygA3ZlM7ZfNE11RxaPDCPthHf3orHZgeBSfCsoBERtvfh9eqc6SdAFSkd8gXRViy38Yt8q0Y/9aRvfz1/sBX2wnUkB8VDYzlZDUr1Aj4BYGetXlgfHPzuD131NtsbcXmU6X7K+xAT6vFLqrnB5uvhPG9w6ZNYxSEOw=,iv:f0kGlZ0dLkOoJg4A2AWnNAnZPBydghI6gyZUR7jmYh4=,tag:KU3J8cGZjrk/xaUokqmuXg==,type:str] 15 + unencrypted_suffix: _unencrypted 16 + version: 3.10.2

+4

infra/_modules/nixos/variables.tf

··· 7 7 ipv6_address = string 8 8 })) 9 9 } 10 + 11 + variable "sops_file" { 12 + type = string 13 + }

+1 -1

infra/_modules/proxmox-vm/variables.tf

··· 17 17 }) 18 18 19 19 default = { 20 - file = "nixos-24.11.20250123.035f8c0-x86_64-linux.iso" 20 + file = "nixos-minimal-25.05.808723.b1b329146965-x86_64-linux.iso" 21 21 } 22 22 } 23 23

+3 -2

infra/production/nixos/terragrunt.hcl

··· 12 12 } 13 13 14 14 inputs = { 15 - flake = "${find_in_parent_folders("_modules")}//nixos" 16 - hosts = dependency.proxmox.outputs.hosts 15 + flake = "${find_in_parent_folders("_modules")}//nixos" 16 + hosts = dependency.proxmox.outputs.hosts 17 + sops_file = find_in_parent_folders("secrets.yaml") 17 18 }

+3 -15

infra/production/proxmox/compute/terragrunt.hcl

··· 9 9 10 10 inputs = { 11 11 hosts = { 12 - "kube-1" = { 13 - cpu = 2 14 - memory = 4 15 - disk = 128 16 - } 17 - "kube-2" = { 18 - cpu = 2 19 - memory = 4 20 - disk = 128 21 - } 22 - "kube-3" = { 23 - cpu = 8 24 - memory = 16 25 - disk = 128 26 - } 12 + "kube-1" = { cpu = 4, memory = 12, disk = 128 } 13 + "kube-2" = { cpu = 4, memory = 12, disk = 128 } 14 + "kube-3" = { cpu = 4, memory = 12, disk = 128 } 27 15 } 28 16 }

+3 -2

infra/production/secrets.yaml

··· 1 + age_key: ENC[AES256_GCM,data:Q5n2nohubYGddLVghlUMUyaL5rgc8sy7fygzkw1Z+7Fk8HSl6nn7IC46OGx3nSu6ypStg1anvjOkTTZWvoL/ETL+pGeRgETdf7Y=,iv:AdSB/+HwVWFD0GQ5kNoybXvGVBLla9CmQ8pAOkf6tow=,tag:ZILB6vg8dXHjptDLtkqTAg==,type:str] 1 2 cloudflare_tfstate_api_token: ENC[AES256_GCM,data:ZNK2qzaiXV93txbV73sSnvxicazxY8uzxf/0yLe/NwQYZKc+t2AVkA==,iv:6gNRl8Kr2xYfwKDVefKP+Fe7/NOf1zKftnBqzoWo5N0=,tag:pjCsHKYL4wbyGGZ63ERvDQ==,type:str] 2 3 cloudflare_tfstate_access_key: ENC[AES256_GCM,data:1N7/YeqYBLaPD0u9EMzQA9nZDpAAk/cb70pF+EwlPMM=,iv:BJ4x220lnXcle3MiDnlj3mY1Wl56z0Fc1bgyA7p294o=,tag:5HcspXjzggy2aQnnR4q71A==,type:str] 3 4 cloudflare_tfstate_secret_key: ENC[AES256_GCM,data:/EEAKxDwtEgMdDNPVXJ5pu8Xzx20ZZwRqMR+90M8KkQwbBRxk67IHP4LeY7Sm6BEQOfjim+tvAdjABfSMpzl4w==,iv:Kj2RP5EDGPkJp5VN7qE6tmCao/QBSxHMRk9VC0up7D4=,tag:fM9oYGx9NpVepSJctp2Y8w==,type:str] ··· 23 24 bzdTeVZWYVIwM1VyUm5UM0pQZW5KYlUKWw7Fq17anbkfY6U3HCr00OKEfffRL/TM 24 25 zvtvDaWLg2ZilP7kWM7YbqyTi4XqjLXKldOblwIcF7qA5E/OfbbzXw== 25 26 -----END AGE ENCRYPTED FILE----- 26 - lastmodified: "2025-08-01T09:36:57Z" 27 - mac: ENC[AES256_GCM,data:kjxsG3hl90Rsu2EQNPwQs+YX5rTaPGV58drChHhwWWQKjxuSxogRBZgCZ7E0+x/8hiU45LAU9cHYASisqS5BQ9OZlHRf6gENWuyEvHA0NXYezO/8Qa+ki0R2/SlrtUNuAk4/v4DUdUydxijTWGL/sF21QwmD4y3e5jrW6+g7inY=,iv:IKxJVRPg9WQdVos6IILnejjKGSSaz/WICahmE2R/OtA=,tag:O56XtwjHQ9Tu4MprEx/LPQ==,type:str] 27 + lastmodified: "2025-08-27T13:36:29Z" 28 + mac: ENC[AES256_GCM,data:BlvMknGkzj83c3WeKgAV+y07xgxiFFfTCB7fM8+5XiYenvv1zSLXrHwxTMk3vWgvlhE5rWu9z/dAfoV10niYuJNTub6k9sk9Bty/q927e2kfwO1xw2N6UA1UROYoJfsFWsFJOiH9E6qMRMP4RwshVNrmpY7ixixWiRgCPTurNCY=,iv:lZ0sVPNhW78dOKaRS87EJG0Zmm2vVkfjS2tjnzTz8qs=,tag:ikaKjIvvLnlolBIdaD0+qQ==,type:str] 28 29 unencrypted_suffix: _unencrypted 29 30 version: 3.10.2

Configure Feed

Configure Feed