agent/activator: wire in switch modes and reboot · adam.robins.wtf/sower@416a25f

+153 -3

apps/sower_agent/lib/sower_agent/deployer.ex

··· 3 3 4 4 alias SowerAgent.Config 5 5 alias SowerAgent.Storage 6 + alias SowerClient.Activator 6 7 alias SowerClient.Orchestration.Deployment 7 8 alias SowerClient.Orchestration.DeploymentProfile 8 9 alias SowerClient.Orchestration.SeedDeployment 9 10 10 11 def run(%Deployment{} = deployment) do 11 - deploy_result = upgrade(deployment) 12 + result = 13 + deployment 14 + |> upgrade() 15 + |> deployment_result() 16 + 17 + maybe_reboot(deployment, result) 18 + result 19 + end 12 20 21 + def deployment_result(deploy_result) do 13 22 Enum.all?(deploy_result, fn r -> 14 23 case r do 15 24 {:ok, {:ok, _}} -> true ··· 84 93 end) 85 94 |> async_stream(fn 86 95 {:ok, {:ok, %SeedDeployment{seed: seed} = seed_deploy}} -> 96 + profile = get_deploy_profile(seed_deploy.subscription_sid) 97 + 87 98 Logger.info( 88 99 msg: "Activating seed", 89 100 name: seed.name, 90 101 seed_sid: seed.sid, 91 102 seed_type: seed.seed_type, 92 103 artifact: seed.artifact, 93 - deployment_sid: deployment.sid 104 + deployment_sid: deployment.sid, 105 + activation_args: get_in(profile.activation_args) 94 106 ) 95 107 96 - result = SowerAgent.Seed.activate(seed, get_deploy_profile(seed_deploy.subscription_sid)) 108 + result = SowerAgent.Seed.activate(seed, profile) 97 109 98 110 case result do 99 111 {:ok, output} -> ··· 173 185 Storage.read().subscriptions |> Enum.find(&(&1.sid == sid)) 174 186 end 175 187 188 + def maybe_reboot(%Deployment{} = _deployment, result) when result != :success, do: :ok 189 + 190 + def maybe_reboot(%Deployment{} = deployment, :success) do 191 + case reboot_reason(deployment.seed_deployments) do 192 + nil -> 193 + :ok 194 + 195 + reason -> 196 + if Application.get_env(:sower_agent, :enable_activation, true) do 197 + Logger.info( 198 + msg: "Reboot required by deployment policy", 199 + deployment_sid: deployment.sid, 200 + reason: reason 201 + ) 202 + 203 + case Activator.reboot(reason: reason) do 204 + {:ok, output} -> 205 + Logger.info( 206 + msg: "Reboot request completed", 207 + deployment_sid: deployment.sid, 208 + reason: reason, 209 + output: output 210 + ) 211 + 212 + {:error, code, output} -> 213 + Logger.error( 214 + msg: "Reboot request failed", 215 + deployment_sid: deployment.sid, 216 + reason: reason, 217 + code: code, 218 + output: output 219 + ) 220 + 221 + {:error, reboot_error} -> 222 + Logger.error( 223 + msg: "Reboot request failed", 224 + deployment_sid: deployment.sid, 225 + reason: reason, 226 + error: inspect(reboot_error) 227 + ) 228 + end 229 + else 230 + Logger.debug( 231 + msg: "Reboot run in noop", 232 + deployment_sid: deployment.sid, 233 + reason: reason 234 + ) 235 + end 236 + end 237 + end 238 + 239 + def reboot_reason( 240 + seed_deployments, 241 + get_profile \\ &get_deploy_profile/1, 242 + read_link \\ &:file.read_link_all/1 243 + ) do 244 + profiles = 245 + seed_deployments 246 + |> Enum.filter(fn %SeedDeployment{seed: seed} -> 247 + seed.seed_type == "nixos" 248 + end) 249 + |> Enum.map(fn %SeedDeployment{subscription_sid: subscription_sid} -> 250 + get_profile.(subscription_sid) || %DeploymentProfile{} 251 + end) 252 + 253 + cond do 254 + profiles == [] -> 255 + nil 256 + 257 + Enum.any?(profiles, fn profile -> 258 + profile.reboot_policy == "always" 259 + end) -> 260 + "policy_always" 261 + 262 + Enum.any?(profiles, fn profile -> 263 + profile.reboot_policy == "when-required" and 264 + SowerAgent.Seed.activation_mode(profile) == "boot" 265 + end) -> 266 + "boot_mode" 267 + 268 + Enum.any?(profiles, fn profile -> 269 + profile.reboot_policy == "when-required" and 270 + SowerAgent.Seed.activation_mode(profile) == "switch" 271 + end) -> 272 + detect_boot_critical_change_reason(read_link) 273 + 274 + true -> 275 + nil 276 + end 277 + end 278 + 176 279 defp maybe_write_log(_deployment, _seed, []), do: :ok 177 280 178 281 # TODO: when you write to disk, you should ensure it gets deleted ··· 194 297 195 298 defp strip_ansi(text) do 196 299 Regex.replace(~r/\x1b\[[0-9;]*[a-zA-Z]/, text, "") 300 + end 301 + 302 + defp detect_boot_critical_change_reason(read_link) do 303 + with {:ok, profile_store_path} <- resolved_symlink("/nix/var/nix/profiles/system", read_link), 304 + {:ok, current_store_path} <- resolved_symlink("/run/current-system", read_link), 305 + {:ok, booted_store_path} <- resolved_symlink("/run/booted-system", read_link) do 306 + cond do 307 + current_store_path != profile_store_path -> 308 + "system_changed" 309 + 310 + "#{current_store_path}/initrd" != "#{booted_store_path}/initrd" -> 311 + "initrd_changed" 312 + 313 + "#{current_store_path}/kernel" != "#{booted_store_path}/kernel" -> 314 + "kernel_changed" 315 + 316 + "#{current_store_path}/kernel-modules" != "#{booted_store_path}/kernel-modules" -> 317 + "modules_changed" 318 + 319 + true -> 320 + nil 321 + end 322 + else 323 + {:error, reason} -> 324 + Logger.warning( 325 + msg: "Could not evaluate reboot requirement from system profile links", 326 + reason: inspect(reason) 327 + ) 328 + 329 + nil 330 + end 331 + end 332 + 333 + defp resolved_symlink(path, read_link) do 334 + case read_link.(path) do 335 + {:ok, resolved} when is_binary(resolved) -> 336 + {:ok, resolved} 337 + 338 + {:ok, resolved} when is_list(resolved) -> 339 + {:ok, List.to_string(resolved)} 340 + 341 + {:error, reason} -> 342 + {:error, {path, reason}} 343 + 344 + other -> 345 + {:error, {path, other}} 346 + end 197 347 end 198 348 end

+17 -3

apps/sower_agent/lib/sower_agent/seed.ex

··· 1 1 defmodule SowerAgent.Seed do 2 2 alias SowerClient.{Activator, Seed} 3 + alias SowerClient.Orchestration.DeploymentProfile 3 4 4 5 require Logger 5 6 6 7 @default_socket_path "/run/sower-activator/activator.sock" 7 8 8 - def activate(%Seed{seed_type: "home-manager"} = seed) do 9 + def activate(seed, profile \\ %DeploymentProfile{}) 10 + 11 + def activate(%Seed{seed_type: "home-manager"} = seed, _profile) do 9 12 run_activation("home-manager", seed.artifact) 10 13 end 11 14 12 - def activate(%Seed{seed_type: "nixos"} = seed) do 13 - run_activation("nixos", seed.artifact, mode: "switch") 15 + def activate(%Seed{seed_type: "nixos"} = seed, %DeploymentProfile{} = profile) do 16 + run_activation("nixos", seed.artifact, mode: activation_mode(profile)) 17 + end 18 + 19 + # TODO pass these args through to the activator once we validate the store paths it receives 20 + def activation_mode(%DeploymentProfile{} = profile) do 21 + case profile.activation_args do 22 + [mode | _] when is_binary(mode) and mode != "" -> 23 + mode 24 + 25 + _ -> 26 + "switch" 27 + end 14 28 end 15 29 16 30 defp run_activation(type, path, opts \\ []) do

+88

apps/sower_agent/test/sower_agent/deployer_test.exs

··· 4 4 import ExUnit.CaptureLog 5 5 6 6 alias SowerAgent.Deployer 7 + alias SowerClient.Orchestration.SeedDeployment 7 8 alias SowerClient.Orchestration.DeploymentProfile 8 9 alias SowerClient.Orchestration.Subscription 10 + alias SowerClient.Seed 9 11 10 12 describe "get_deploy_profile/3" do 11 13 test "returns nil for nil subscription sid" do ··· 81 83 fn _ -> nil end 82 84 ) == %DeploymentProfile{} 83 85 end 86 + end 87 + 88 + describe "deployment_result/1" do 89 + test "returns :success when all seed activations succeed" do 90 + result = [{:ok, {:ok, ["ok"]}}, {:ok, {:ok, ["ok"]}}] 91 + assert Deployer.deployment_result(result) == :success 92 + end 93 + 94 + test "returns :partial when some seed activations fail" do 95 + result = [{:ok, {:ok, ["ok"]}}, {:ok, {:error, 1, ["failed"]}}] 96 + assert Deployer.deployment_result(result) == :partial 97 + end 98 + 99 + test "returns :failure when all seed activations fail" do 100 + result = [{:ok, {:error, 1, ["failed"]}}, {:error, :failed_to_realize, %{}}] 101 + assert Deployer.deployment_result(result) == :failure 102 + end 103 + end 104 + 105 + describe "reboot_reason/3" do 106 + test "returns nil when there are no nixos seed deployments" do 107 + seed_deployments = [seed_deploy("sub1", "home-manager")] 108 + 109 + assert Deployer.reboot_reason(seed_deployments, fn _ -> %DeploymentProfile{} end) == nil 110 + end 111 + 112 + test "returns policy_always when profile reboot policy is always" do 113 + seed_deployments = [seed_deploy("sub_always")] 114 + 115 + get_profile = fn "sub_always" -> 116 + %DeploymentProfile{activation_args: ["switch"], reboot_policy: "always"} 117 + end 118 + 119 + assert Deployer.reboot_reason(seed_deployments, get_profile) == "policy_always" 120 + end 121 + 122 + test "returns boot_mode when profile is when-required and activation mode is boot" do 123 + seed_deployments = [seed_deploy("sub_boot")] 124 + 125 + get_profile = fn "sub_boot" -> 126 + %DeploymentProfile{activation_args: ["boot"], reboot_policy: "when-required"} 127 + end 128 + 129 + assert Deployer.reboot_reason(seed_deployments, get_profile) == "boot_mode" 130 + end 131 + 132 + test "returns initrd_changed when when-required switch profile has boot-critical changes" do 133 + seed_deployments = [seed_deploy("sub_switch")] 134 + 135 + get_profile = fn "sub_switch" -> 136 + %DeploymentProfile{activation_args: ["switch"], reboot_policy: "when-required"} 137 + end 138 + 139 + read_link = fn 140 + "/nix/var/nix/profiles/system" -> {:ok, "/nix/store/sys-a"} 141 + "/run/current-system" -> {:ok, "/nix/store/sys-a"} 142 + "/run/booted-system" -> {:ok, "/nix/store/sys-b"} 143 + end 144 + 145 + assert Deployer.reboot_reason(seed_deployments, get_profile, read_link) == "initrd_changed" 146 + end 147 + 148 + test "returns nil and logs warning when boot-critical detection cannot read links" do 149 + seed_deployments = [seed_deploy("sub_switch")] 150 + 151 + get_profile = fn "sub_switch" -> 152 + %DeploymentProfile{activation_args: ["switch"], reboot_policy: "when-required"} 153 + end 154 + 155 + logs = 156 + capture_log(fn -> 157 + assert Deployer.reboot_reason(seed_deployments, get_profile, fn _ -> 158 + {:error, :enoent} 159 + end) == 160 + nil 161 + end) 162 + 163 + assert logs =~ "Could not evaluate reboot requirement from system profile links" 164 + end 165 + end 166 + 167 + defp seed_deploy(subscription_sid, seed_type \\ "nixos") do 168 + %SeedDeployment{ 169 + subscription_sid: subscription_sid, 170 + seed: %Seed{seed_type: seed_type} 171 + } 84 172 end 85 173 end

+32

apps/sower_agent/test/sower_agent/seed_test.exs

··· 4 4 import ExUnit.CaptureLog 5 5 6 6 alias SowerAgent.Seed 7 + alias SowerClient.Orchestration.DeploymentProfile 7 8 alias SowerClient.Seed, as: ClientSeed 8 9 9 10 describe "activate/1" do ··· 93 94 end) 94 95 95 96 assert log =~ "Failed to activate" 97 + end 98 + end 99 + 100 + describe "activate/2" do 101 + test "uses deployment profile activation_args for nixos mode" do 102 + {socket_path, server_pid} = 103 + start_mock_server(fn request_line, client_socket -> 104 + request = Jason.decode!(request_line) 105 + assert request["type"] == "nixos" 106 + assert request["path"] == "/nix/store/xyz" 107 + assert request["mode"] == "boot" 108 + 109 + send_response(client_socket, %{id: request["id"], type: "complete", exit_code: 0}) 110 + end) 111 + 112 + Application.put_env(:sower_agent, :activator_socket, socket_path) 113 + 114 + on_exit(fn -> 115 + Application.delete_env(:sower_agent, :activator_socket) 116 + stop_mock_server(server_pid) 117 + File.rm_rf!(Path.dirname(socket_path)) 118 + end) 119 + 120 + seed = %ClientSeed{name: "test", seed_type: "nixos", artifact: "/nix/store/xyz"} 121 + 122 + profile = %DeploymentProfile{ 123 + activation_args: ["boot"], 124 + reboot_policy: "never" 125 + } 126 + 127 + assert {:ok, []} = Seed.activate(seed, profile) 96 128 end 97 129 end 98 130

+73 -3

apps/sower_client/lib/sower_client/activator.ex

··· 20 20 typedstruct module: Request do 21 21 field(:id, String.t(), enforce: true) 22 22 field(:type, String.t(), enforce: true) 23 - field(:path, String.t(), enforce: true) 23 + field(:path, String.t()) 24 24 field(:mode, String.t()) 25 + field(:reason, String.t()) 25 26 end 26 27 27 28 typedstruct module: Response do ··· 60 61 end 61 62 62 63 @doc """ 64 + Request a system reboot via socket or CLI fallback. 65 + 66 + ## Options 67 + 68 + - `:socket_path` - Path to activator socket (default: #{@default_socket_path}) 69 + - `:reason` - Optional reason attached to reboot request 70 + - `:on_output` - Callback function for streaming output 71 + """ 72 + def reboot(opts \\ []) do 73 + socket_path = Keyword.get(opts, :socket_path, @default_socket_path) 74 + 75 + if socket_available?(socket_path) do 76 + reboot_via_socket(opts) 77 + else 78 + Logger.debug("Socket not available, falling back to CLI reboot") 79 + reboot_via_cli(opts) 80 + end 81 + end 82 + 83 + @doc """ 63 84 Activate via Unix socket connection to sower-activator daemon. 64 85 65 86 ## Options ··· 95 116 end 96 117 97 118 @doc """ 119 + Request reboot via Unix socket connection to sower-activator daemon. 120 + """ 121 + def reboot_via_socket(opts \\ []) do 122 + socket_path = Keyword.get(opts, :socket_path, @default_socket_path) 123 + reason = Keyword.get(opts, :reason) 124 + on_output = Keyword.get(opts, :on_output, fn _line -> :ok end) 125 + 126 + request = %Request{ 127 + id: generate_request_id(), 128 + type: "reboot", 129 + reason: reason 130 + } 131 + 132 + with {:ok, socket} <- connect(socket_path), 133 + :ok <- send_request(socket, request), 134 + result <- receive_responses(socket, request.id, on_output) do 135 + :gen_tcp.close(socket) 136 + result 137 + end 138 + end 139 + 140 + @doc """ 98 141 Activate via CLI invocation of sower-activator binary. 99 142 100 143 Requires `sudo` and `sower-activator` in PATH. Invokes: ··· 136 179 end 137 180 138 181 @doc """ 182 + Request reboot via CLI invocation of `systemctl reboot`. 183 + """ 184 + def reboot_via_cli(_opts \\ []) do 185 + with systemctl when not is_nil(systemctl) <- System.find_executable("systemctl"), 186 + sudo when not is_nil(sudo) <- System.find_executable("sudo") do 187 + case System.cmd(sudo, [systemctl, "reboot"], 188 + into: [], 189 + lines: 1024, 190 + stderr_to_stdout: true 191 + ) do 192 + {output, 0} -> 193 + Logger.debug(output: output) 194 + {:ok, output} 195 + 196 + {output, code} -> 197 + Logger.error(msg: "Reboot failed", output: output, return_code: code) 198 + {:error, code, output} 199 + end 200 + else 201 + nil -> 202 + Logger.error("Required executables not found: sudo and/or systemctl") 203 + {:error, :cmd_not_found} 204 + end 205 + end 206 + 207 + @doc """ 139 208 Check if activator socket exists and is connectable. 140 209 141 210 Returns `true` if socket file exists, `false` otherwise. ··· 190 259 map = 191 260 %{ 192 261 "id" => request.id, 193 - "type" => request.type, 194 - "path" => request.path 262 + "type" => request.type 195 263 } 264 + |> maybe_put("path", request.path) 196 265 |> maybe_put("mode", request.mode) 266 + |> maybe_put("reason", request.reason) 197 267 198 268 Jason.encode!(map) 199 269 end

+45

apps/sower_client/test/sower_client/activator_test.exs

··· 96 96 end 97 97 end 98 98 99 + describe "reboot_via_socket/1" do 100 + test "sends reboot request with reason" do 101 + {socket_path, server_pid} = 102 + start_mock_server(fn request_line, client_socket -> 103 + request = Jason.decode!(request_line) 104 + assert request["type"] == "reboot" 105 + assert request["reason"] == "policy_always" 106 + refute Map.has_key?(request, "path") 107 + refute Map.has_key?(request, "mode") 108 + 109 + send_response(client_socket, %{id: request["id"], type: "complete", exit_code: 0}) 110 + end) 111 + 112 + on_exit(fn -> 113 + stop_mock_server(server_pid) 114 + File.rm_rf!(Path.dirname(socket_path)) 115 + end) 116 + 117 + assert {:ok, []} = 118 + Activator.reboot_via_socket( 119 + socket_path: socket_path, 120 + reason: "policy_always" 121 + ) 122 + end 123 + end 124 + 99 125 describe "activate_via_cli/3" do 100 126 test "returns error when executables not found" do 101 127 # Ensure the executables don't exist in PATH ··· 142 168 capture_log(fn -> 143 169 result = 144 170 Activator.activate("nixos", "/nix/store/xyz", socket_path: "/nonexistent/socket") 171 + 172 + assert {:error, :cmd_not_found} = result 173 + end) 174 + end 175 + end 176 + 177 + describe "reboot/1" do 178 + test "falls back to CLI when socket is not available" do 179 + original_path = System.get_env("PATH") 180 + System.put_env("PATH", "/nonexistent") 181 + 182 + on_exit(fn -> System.put_env("PATH", original_path) end) 183 + 184 + capture_log(fn -> 185 + result = 186 + Activator.reboot( 187 + socket_path: "/nonexistent/socket", 188 + reason: "policy_always" 189 + ) 145 190 146 191 assert {:error, :cmd_not_found} = result 147 192 end)

+8

cmd/sower-activator/activate.go

··· 102 102 } 103 103 } 104 104 105 + // rebootStreaming requests a system reboot and streams output via callback. 106 + func rebootStreaming(callback OutputCallback) (int, error) { 107 + args := []string{"reboot"} 108 + 109 + cmd := exec.Command("systemctl", args...) 110 + return runCommandStreaming(cmd, callback) 111 + } 112 + 105 113 // runCommandStreaming executes a command and streams output via callback. 106 114 // Returns the exit code and any error that occurred. 107 115 func runCommandStreaming(cmd *exec.Cmd, callback OutputCallback) (int, error) {

+35 -9

cmd/sower-activator/handler.go

··· 59 59 return 60 60 } 61 61 62 - var req ActivateRequest 62 + var req Request 63 63 if err := json.Unmarshal(line, &req); err != nil { 64 64 slog.Error("Failed to parse request", "error", err) 65 65 h.sendError("", "invalid JSON") 66 66 return 67 67 } 68 68 69 - slog.Info("Received activation request", "id", req.ID, "type", req.Type, "path", req.Path, "mode", req.Mode) 69 + slog.Info( 70 + "Received request", 71 + "id", 72 + req.ID, 73 + "type", 74 + req.Type, 75 + "path", 76 + req.Path, 77 + "mode", 78 + req.Mode, 79 + "reason", 80 + req.Reason, 81 + ) 70 82 71 83 // Validate request 72 84 if err := h.validateRequest(&req); err != nil { ··· 75 87 return 76 88 } 77 89 78 - // Execute activation with streaming output 79 - exitCode := h.executeActivation(&req) 90 + // Execute request with streaming output 91 + exitCode := h.executeRequest(&req) 80 92 h.sendComplete(req.ID, exitCode) 81 93 } 82 94 83 95 // validateRequest checks that the request is valid. 84 - func (h *ConnectionHandler) validateRequest(req *ActivateRequest) error { 96 + func (h *ConnectionHandler) validateRequest(req *Request) error { 85 97 if req.ID == "" { 86 98 return fmt.Errorf("missing request ID") 87 99 } 88 100 101 + if req.Type == "reboot" { 102 + return nil 103 + } 104 + 89 105 if req.Type != SeedTypeNixOS && req.Type != SeedTypeHomeManager { 90 106 return fmt.Errorf("invalid type: %s", req.Type) 91 107 } ··· 113 129 return nil 114 130 } 115 131 116 - // executeActivation runs the activation and streams output. 117 - func (h *ConnectionHandler) executeActivation(req *ActivateRequest) int { 132 + // executeRequest runs the request action and streams output. 133 + func (h *ConnectionHandler) executeRequest(req *Request) int { 118 134 outputCallback := func(line string, isError bool) { 119 135 respType := ResponseTypeOutput 120 136 if isError { ··· 127 143 }) 128 144 } 129 145 130 - exitCode, err := activateStreaming(req.Type, req.Path, req.Mode, outputCallback) 146 + var ( 147 + exitCode int 148 + err error 149 + ) 150 + 151 + if req.Type == "reboot" { 152 + exitCode, err = rebootStreaming(outputCallback) 153 + } else { 154 + exitCode, err = activateStreaming(req.Type, req.Path, req.Mode, outputCallback) 155 + } 156 + 131 157 if err != nil { 132 - slog.Error("Activation failed", "id", req.ID, "error", err) 158 + slog.Error("Request failed", "id", req.ID, "type", req.Type, "error", err) 133 159 h.sendResponse(ActivateResponse{ 134 160 ID: req.ID, 135 161 Type: ResponseTypeError,

+7 -6

cmd/sower-activator/protocol.go

··· 1 1 package main 2 2 3 - // ActivateRequest is sent by clients to request activation. 4 - type ActivateRequest struct { 5 - ID string `json:"id"` // Request correlation ID 6 - Type string `json:"type"` // "nixos" or "home-manager" 7 - Path string `json:"path"` // Nix store path 8 - Mode string `json:"mode"` // "switch", "boot", etc. (NixOS only) 3 + // Request is sent by clients to request activation or reboot. 4 + type Request struct { 5 + ID string `json:"id"` // Request correlation ID 6 + Type string `json:"type"` // "nixos", "home-manager", or "reboot" 7 + Path string `json:"path,omitempty"` // Nix store path for activation 8 + Mode string `json:"mode,omitempty"` // "switch", "boot", etc. (NixOS only) 9 + Reason string `json:"reason,omitempty"` // Reboot reason 9 10 } 10 11 11 12 // ResponseType indicates the type of response message.

Configure Feed

Configure Feed