Deployment and lifecycle management for Nix
nix: add upload to cache capability
+356
+63
apps/nix/lib/nix/cache.ex
+63
apps/nix/lib/nix/cache.ex
···
1
+
defmodule Nix.Cache do
2
+
@moduledoc """
3
+
Behavior for binary cache upload backends.
4
+
"""
5
+
6
+
@doc """
7
+
Upload a batch of store paths to the configured cache.
8
+
9
+
Returns `{:ok, result}` where result contains:
10
+
- `:uploaded` - List of successfully uploaded paths
11
+
- `:failed` - List of tuples `{path, reason}` for failures
12
+
13
+
All paths in the input should appear in either `uploaded` or `failed`.
14
+
15
+
For backends that upload all-or-nothing, a failure means all paths
16
+
are in the `failed` list, while success means all are in `uploaded`.
17
+
18
+
## Examples
19
+
20
+
{:ok, %{uploaded: ["/nix/store/abc-foo", "/nix/store/xyz-bar"], failed: []}}
21
+
22
+
{:ok, %{uploaded: ["/nix/store/abc-foo"], failed: [{"/nix/store/xyz-bar", "permission denied"}]}}
23
+
24
+
{:error, "invalid destination"}
25
+
"""
26
+
@callback upload(paths :: [String.t()] | String.t(), config :: map()) ::
27
+
{:ok, %{uploaded: [String.t()], failed: [{String.t(), term()}]}}
28
+
| {:error, term()}
29
+
30
+
@doc """
31
+
Validate the backend configuration.
32
+
33
+
Called before upload starts to fail fast on invalid config.
34
+
35
+
## Examples
36
+
37
+
:ok
38
+
39
+
{:error, "destination required"}
40
+
"""
41
+
@callback validate_config(config :: map()) :: :ok | {:error, term()}
42
+
43
+
@doc """
44
+
Return a human-readable name for this backend.
45
+
46
+
Used for logging and error messages.
47
+
"""
48
+
@callback name() :: String.t()
49
+
50
+
defmacro __using__(_opts) do
51
+
quote do
52
+
@behaviour Nix.Cache
53
+
54
+
defp ensure_store_paths(paths) when is_list(paths) do
55
+
paths
56
+
|> Enum.map(fn
57
+
path when is_binary(path) -> path
58
+
path when is_struct(path, Nix.Build) -> path.store_path
59
+
end)
60
+
end
61
+
end
62
+
end
63
+
end
+175
apps/nix/lib/nix/cache/attic.ex
+175
apps/nix/lib/nix/cache/attic.ex
···
1
+
defmodule Nix.Cache.Attic do
2
+
@moduledoc """
3
+
Binary cache backend using Attic.
4
+
5
+
Attic is a Nix binary cache server with efficient uploads and built-in
6
+
parallelism. See: https://github.com/zhaofengli/attic
7
+
8
+
## Configuration
9
+
10
+
%{
11
+
cache: "server:cache-name", # Format: "server-name:cache-name"
12
+
jobs: 5 # Optional: parallel upload jobs (default: 5)
13
+
}
14
+
15
+
The server and cache must be configured in `~/.config/attic/config.toml`:
16
+
17
+
[cache."server:cache-name"]
18
+
endpoint = "https://cache.example.com"
19
+
token = "your-auth-token"
20
+
21
+
## Example
22
+
23
+
config = %{cache: "production:my-cache", jobs: 8}
24
+
{:ok, result} = Nix.Cache.Attic.upload(config, [
25
+
"/nix/store/abc123-foo",
26
+
"/nix/store/xyz789-bar"
27
+
])
28
+
"""
29
+
30
+
use Nix.Cache
31
+
32
+
require Logger
33
+
34
+
@impl Nix.Cache
35
+
def name(), do: "attic"
36
+
37
+
@impl Nix.Cache
38
+
def validate_config(%{cache: cache}) when is_binary(cache) and byte_size(cache) > 0 do
39
+
# Validate cache format: "server:cache-name"
40
+
if String.contains?(cache, ":") do
41
+
:ok
42
+
else
43
+
{:error, "cache must be in format 'server:cache-name'"}
44
+
end
45
+
end
46
+
47
+
def validate_config(_) do
48
+
{:error, "cache required (format: 'server:cache-name')"}
49
+
end
50
+
51
+
@impl Nix.Cache
52
+
def upload(path, config) when not is_list(path), do: upload([path], config)
53
+
54
+
def upload(paths, config) when is_list(paths) do
55
+
if length(paths) == 0 do
56
+
{:ok, %{uploaded: [], failed: []}}
57
+
else
58
+
paths = ensure_store_paths(paths)
59
+
60
+
%{cache: cache} = config
61
+
jobs = Map.get(config, :jobs, 5)
62
+
63
+
attic_cmd = System.find_executable("attic")
64
+
65
+
if is_nil(attic_cmd) do
66
+
{:error, "attic command not found in PATH"}
67
+
else
68
+
cmd = [attic_cmd, "push", "--stdin", "-j", to_string(jobs), cache]
69
+
stdin_input = Enum.join(paths, "\n")
70
+
71
+
Logger.debug(
72
+
msg: "Uploading to cache",
73
+
backend: "attic",
74
+
cache: cache,
75
+
path_count: length(paths),
76
+
jobs: jobs
77
+
)
78
+
79
+
case run_with_stdin(cmd, stdin_input) do
80
+
{:ok, _result} ->
81
+
Logger.info(
82
+
msg: "Upload succeeded",
83
+
backend: "attic",
84
+
cache: cache,
85
+
path_count: length(paths)
86
+
)
87
+
88
+
{:ok, %{uploaded: paths, failed: []}}
89
+
90
+
{:error, %{exit_status: exit_code} = result} ->
91
+
output = Map.get(result, :stderr, "") <> Map.get(result, :stdout, "")
92
+
93
+
Logger.error(
94
+
msg: "Upload failed",
95
+
backend: "attic",
96
+
cache: cache,
97
+
exit_code: exit_code,
98
+
output: String.slice(output, 0, 500)
99
+
)
100
+
101
+
error_reason = parse_error(output, exit_code)
102
+
{:error, error_reason}
103
+
end
104
+
end
105
+
end
106
+
end
107
+
108
+
defp run_with_stdin(cmd, stdin_data) do
109
+
case :exec.run(cmd, [:stdin, :stdout, :stderr, :monitor]) do
110
+
{:ok, _pid, ospid} ->
111
+
:ok = :exec.send(ospid, stdin_data)
112
+
:ok = :exec.send(ospid, :eof)
113
+
await_completion(ospid, [], [])
114
+
115
+
{:error, reason} ->
116
+
{:error, %{exit_status: 1, stderr: to_string(reason), stdout: ""}}
117
+
end
118
+
end
119
+
120
+
defp await_completion(ospid, stdout, stderr) do
121
+
receive do
122
+
{:stdout, ^ospid, data} ->
123
+
await_completion(ospid, [data | stdout], stderr)
124
+
125
+
{:stderr, ^ospid, data} ->
126
+
await_completion(ospid, stdout, [data | stderr])
127
+
128
+
{:DOWN, _ospid, :process, _pid, :normal} ->
129
+
{:ok, finalize_output(stdout, stderr)}
130
+
131
+
{:DOWN, _ospid, :process, _pid, {:exit_status, status}} ->
132
+
{:error, finalize_output(stdout, stderr) |> Map.put(:exit_status, status)}
133
+
after
134
+
# 30 minute timeout for large uploads
135
+
30 * 60 * 1000 ->
136
+
:exec.kill(ospid, :sigterm)
137
+
{:error, %{exit_status: 1, stderr: "timeout", stdout: ""}}
138
+
end
139
+
end
140
+
141
+
defp finalize_output(stdout, stderr) do
142
+
%{
143
+
stdout: stdout |> Enum.reverse() |> IO.iodata_to_binary(),
144
+
stderr: stderr |> Enum.reverse() |> IO.iodata_to_binary()
145
+
}
146
+
end
147
+
148
+
defp parse_error(output, exit_code) do
149
+
cond do
150
+
String.contains?(output, "401 Unauthorized") or
151
+
String.contains?(output, "permission denied") ->
152
+
"authentication failed - check token in attic config"
153
+
154
+
String.contains?(output, "404 Not Found") ->
155
+
"cache not found"
156
+
157
+
String.contains?(output, "Connection refused") or
158
+
String.contains?(output, "failed to connect") ->
159
+
"connection refused - check cache endpoint"
160
+
161
+
String.contains?(output, "No cache named") ->
162
+
"cache not configured in ~/.config/attic/config.toml"
163
+
164
+
true ->
165
+
# Return exit code and first line of output
166
+
first_line =
167
+
output
168
+
|> String.split("\n", parts: 2)
169
+
|> List.first()
170
+
|> String.slice(0, 200)
171
+
172
+
{exit_code, first_line}
173
+
end
174
+
end
175
+
end
+118
apps/nix/lib/nix/cache/nix_copy.ex
+118
apps/nix/lib/nix/cache/nix_copy.ex
···
1
+
defmodule Nix.Cache.NixCopy do
2
+
@moduledoc """
3
+
Binary cache backend using `nix copy`.
4
+
5
+
Supports uploading to various destinations:
6
+
- SSH: `ssh://user@host`
7
+
- S3: `s3://bucket-name` (requires AWS credentials configured)
8
+
- HTTP: `http://cache.example.com` or `https://cache.example.com`
9
+
- Local: `file:///path/to/cache`
10
+
11
+
## Configuration
12
+
13
+
%{destination: "ssh://user@host/nix-cache"}
14
+
%{destination: "s3://my-bucket"}
15
+
%{destination: "https://cache.example.com"}
16
+
17
+
## Example
18
+
19
+
config = %{destination: "ssh://builder@cache.example.com"}
20
+
{:ok, result} = Nix.Cache.NixCopy.upload(config, [
21
+
"/nix/store/abc123-foo",
22
+
"/nix/store/xyz789-bar"
23
+
])
24
+
"""
25
+
26
+
use Nix.Cache
27
+
28
+
require Logger
29
+
30
+
@impl Nix.Cache
31
+
def name(), do: "nix copy"
32
+
33
+
@impl Nix.Cache
34
+
def validate_config(%{destination: dest}) when is_binary(dest) and byte_size(dest) > 0 do
35
+
:ok
36
+
end
37
+
38
+
def validate_config(_) do
39
+
{:error, "destination required (e.g., ssh://host, s3://bucket, https://cache)"}
40
+
end
41
+
42
+
@impl Nix.Cache
43
+
def upload(path, config) when not is_list(path), do: upload([path], config)
44
+
45
+
def upload(paths, %{destination: dest}) when is_list(paths) do
46
+
if length(paths) == 0 do
47
+
{:ok, %{uploaded: [], failed: []}}
48
+
else
49
+
nix_cmd = System.find_executable("nix")
50
+
paths = ensure_store_paths(paths)
51
+
52
+
if is_nil(nix_cmd) do
53
+
{:error, "nix command not found in PATH"}
54
+
else
55
+
args = ["copy", "--to", dest] ++ paths
56
+
57
+
Logger.debug(
58
+
msg: "Uploading to cache",
59
+
backend: "nix copy",
60
+
destination: dest,
61
+
path_count: length(paths)
62
+
)
63
+
64
+
case System.cmd(nix_cmd, args, stderr_to_stdout: true) do
65
+
{_output, 0} ->
66
+
Logger.info(
67
+
msg: "Upload succeeded",
68
+
backend: "nix copy",
69
+
destination: dest,
70
+
path_count: length(paths)
71
+
)
72
+
73
+
{:ok, %{uploaded: paths, failed: []}}
74
+
75
+
{output, exit_code} ->
76
+
# nix copy is all-or-nothing - if it fails, all paths failed
77
+
Logger.error(
78
+
msg: "Upload failed",
79
+
backend: "nix copy",
80
+
destination: dest,
81
+
exit_code: exit_code,
82
+
output: String.slice(output, 0, 500)
83
+
)
84
+
85
+
error_reason = parse_error(output, exit_code)
86
+
{:error, error_reason}
87
+
end
88
+
end
89
+
end
90
+
end
91
+
92
+
defp parse_error(output, exit_code) do
93
+
cond do
94
+
String.contains?(output, "permission denied") or
95
+
String.contains?(output, "Permission denied") ->
96
+
"permission denied"
97
+
98
+
String.contains?(output, "No such file or directory") ->
99
+
"destination not found"
100
+
101
+
String.contains?(output, "Connection refused") ->
102
+
"connection refused"
103
+
104
+
String.contains?(output, "Host key verification failed") ->
105
+
"SSH host key verification failed"
106
+
107
+
true ->
108
+
# Return exit code and first line of output
109
+
first_line =
110
+
output
111
+
|> String.split("\n", parts: 2)
112
+
|> List.first()
113
+
|> String.slice(0, 200)
114
+
115
+
{exit_code, first_line}
116
+
end
117
+
end
118
+
end