A fork of attic a self-hostable Nix Binary Cache server
upload-path: Support including the upload info as part of the PUT body
Often times there are tight limits on how large headers can be.
+132
-30
+11
-2
attic/src/api/v1/upload_path.rs
+11
-2
attic/src/api/v1/upload_path.rs
···
8
8
/// Header containing the upload info.
9
9
pub const ATTIC_NAR_INFO: &str = "X-Attic-Nar-Info";
10
10
11
+
/// Header containing the size of the upload info at the beginning of the body.
12
+
pub const ATTIC_NAR_INFO_PREAMBLE_SIZE: &str = "X-Attic-Nar-Info-Preamble-Size";
13
+
11
14
/// NAR information associated with a upload.
12
15
///
13
-
/// This is JSON-serialized as the value of the `X-Attic-Nar-Info` header.
14
-
/// The (client-compressed) NAR is the PUT body.
16
+
/// There are two ways for the client to supply the NAR information:
17
+
///
18
+
/// 1. At the beginning of the PUT body. The `X-Attic-Nar-Info-Preamble-Size`
19
+
/// header must be set to the size of the JSON.
20
+
/// 2. Through the `X-Attic-Nar-Info` header.
21
+
///
22
+
/// The client is advised to use the first method if the serialized
23
+
/// JSON is large (>4K).
15
24
///
16
25
/// Regardless of client compression, the server will always decompress
17
26
/// the NAR to validate the NAR hash before applying the server-configured
+30
-9
client/src/api/mod.rs
+30
-9
client/src/api/mod.rs
···
5
5
use bytes::Bytes;
6
6
use const_format::concatcp;
7
7
use displaydoc::Display;
8
-
use futures::TryStream;
8
+
use futures::{
9
+
future,
10
+
stream::{self, StreamExt, TryStream, TryStreamExt},
11
+
};
9
12
use reqwest::{
10
13
header::{HeaderMap, HeaderValue, AUTHORIZATION, USER_AGENT},
11
14
Body, Client as HttpClient, Response, StatusCode, Url,
···
16
19
use crate::version::ATTIC_DISTRIBUTOR;
17
20
use attic::api::v1::cache_config::{CacheConfig, CreateCacheRequest};
18
21
use attic::api::v1::get_missing_paths::{GetMissingPathsRequest, GetMissingPathsResponse};
19
-
use attic::api::v1::upload_path::{UploadPathNarInfo, UploadPathResult, ATTIC_NAR_INFO};
22
+
use attic::api::v1::upload_path::{
23
+
UploadPathNarInfo, UploadPathResult, ATTIC_NAR_INFO, ATTIC_NAR_INFO_PREAMBLE_SIZE,
24
+
};
20
25
use attic::cache::CacheName;
21
26
use attic::nix_store::StorePathHash;
22
27
23
28
/// The User-Agent string of Attic.
24
29
const ATTIC_USER_AGENT: &str =
25
30
concatcp!("Attic/{} ({})", env!("CARGO_PKG_NAME"), ATTIC_DISTRIBUTOR);
31
+
32
+
/// The size threshold to send the upload info as part of the PUT body.
33
+
const NAR_INFO_PREAMBLE_THRESHOLD: usize = 4 * 1024; // 4 KiB
26
34
27
35
/// The Attic API client.
28
36
#[derive(Debug, Clone)]
···
165
173
&self,
166
174
nar_info: UploadPathNarInfo,
167
175
stream: S,
176
+
force_preamble: bool,
168
177
) -> Result<Option<UploadPathResult>>
169
178
where
170
-
S: TryStream + Send + Sync + 'static,
171
-
S::Error: Into<Box<dyn StdError + Send + Sync>>,
172
-
Bytes: From<S::Ok>,
179
+
S: TryStream<Ok = Bytes> + Send + Sync + 'static,
180
+
S::Error: Into<Box<dyn StdError + Send + Sync>> + Send + Sync,
173
181
{
174
182
let endpoint = self.endpoint.join("_api/v1/upload-path")?;
175
183
let upload_info_json = serde_json::to_string(&nar_info)?;
176
184
177
-
let req = self
185
+
let mut req = self
178
186
.client
179
187
.put(endpoint)
180
-
.header(ATTIC_NAR_INFO, HeaderValue::from_str(&upload_info_json)?)
181
-
.header(USER_AGENT, HeaderValue::from_str(ATTIC_USER_AGENT)?)
182
-
.body(Body::wrap_stream(stream));
188
+
.header(USER_AGENT, HeaderValue::from_str(ATTIC_USER_AGENT)?);
189
+
190
+
if force_preamble || upload_info_json.len() >= NAR_INFO_PREAMBLE_THRESHOLD {
191
+
let preamble = Bytes::from(upload_info_json);
192
+
let preamble_len = preamble.len();
193
+
let preamble_stream = stream::once(future::ok(preamble));
194
+
195
+
let chained = preamble_stream.chain(stream.into_stream());
196
+
req = req
197
+
.header(ATTIC_NAR_INFO_PREAMBLE_SIZE, preamble_len)
198
+
.body(Body::wrap_stream(chained));
199
+
} else {
200
+
req = req
201
+
.header(ATTIC_NAR_INFO, HeaderValue::from_str(&upload_info_json)?)
202
+
.body(Body::wrap_stream(stream));
203
+
}
183
204
184
205
let res = req.send().await?;
185
206
+22
-4
client/src/command/push.rs
+22
-4
client/src/command/push.rs
···
7
7
use std::time::{Duration, Instant};
8
8
9
9
use anyhow::{anyhow, Result};
10
+
use bytes::Bytes;
10
11
use clap::Parser;
11
12
use futures::future::join_all;
12
-
use futures::stream::Stream;
13
+
use futures::stream::{Stream, TryStreamExt};
13
14
use indicatif::{HumanBytes, MultiProgress, ProgressBar, ProgressState, ProgressStyle};
14
15
use tokio::sync::Semaphore;
15
16
···
41
42
/// The maximum number of parallel upload processes.
42
43
#[clap(short = 'j', long, default_value = "5")]
43
44
jobs: usize,
45
+
46
+
/// Always send the upload info as part of the payload.
47
+
#[clap(long, hide = true)]
48
+
force_preamble: bool,
44
49
}
45
50
46
51
struct PushPlan {
···
70
75
api: ApiClient,
71
76
cache: &CacheName,
72
77
mp: MultiProgress,
78
+
force_preamble: bool,
73
79
) -> Result<()> {
74
80
let path = &path_info.path;
75
81
let upload_info = {
···
127
133
);
128
134
let bar = mp.add(ProgressBar::new(path_info.nar_size));
129
135
bar.set_style(style);
130
-
let nar_stream = NarStreamProgress::new(store.nar_from_path(path.to_owned()), bar.clone());
136
+
let nar_stream = NarStreamProgress::new(store.nar_from_path(path.to_owned()), bar.clone())
137
+
.map_ok(Bytes::from);
131
138
132
139
let start = Instant::now();
133
-
match api.upload_path(upload_info, nar_stream).await {
140
+
match api
141
+
.upload_path(upload_info, nar_stream, force_preamble)
142
+
.await
143
+
{
134
144
Ok(r) => {
135
145
let r = r.unwrap_or(UploadPathResult {
136
146
kind: UploadPathResultKind::Uploaded,
···
243
253
async move {
244
254
let permit = upload_limit.acquire().await?;
245
255
246
-
upload_path(store.clone(), path_info, api, cache, mp.clone()).await?;
256
+
upload_path(
257
+
store.clone(),
258
+
path_info,
259
+
api,
260
+
cache,
261
+
mp.clone(),
262
+
sub.force_preamble,
263
+
)
264
+
.await?;
247
265
248
266
drop(permit);
249
267
Ok::<(), anyhow::Error>(())
+19
-5
integration-tests/basic/default.nix
+19
-5
integration-tests/basic/default.nix
···
9
9
atticd = ". /etc/atticd.env && export ATTIC_SERVER_TOKEN_HS256_SECRET_BASE64 && atticd -f ${serverConfigFile}";
10
10
};
11
11
12
-
testDrv = pkgs.writeText "test.nix" ''
12
+
makeTestDerivation = pkgs.writeShellScript "make-drv" ''
13
+
name=$1
14
+
base=$(basename $name)
15
+
16
+
cat >$name <<EOF
13
17
#!/bin/sh
14
-
/*/sh -c "echo hello > $out"; exit 0; */
18
+
/*/sh -c "echo hello > \$out"; exit 0; */
15
19
derivation {
16
-
name = "hello.txt";
17
-
builder = ./test.nix;
20
+
name = "$base";
21
+
builder = ./$name;
18
22
system = builtins.currentSystem;
19
23
preferLocalBuild = true;
20
24
allowSubstitutes = false;
21
25
}
26
+
EOF
27
+
28
+
chmod +x $name
22
29
'';
23
30
24
31
databaseModules = {
···
171
178
client.succeed("attic cache create test")
172
179
173
180
with subtest("Check that we can push a path"):
174
-
client.succeed("cat ${testDrv} >test.nix && chmod +x test.nix")
181
+
client.succeed("${makeTestDerivation} test.nix")
175
182
test_file = client.succeed("nix-build --no-out-link test.nix")
176
183
test_file_hash = test_file.removeprefix("/nix/store/")[:32]
177
184
···
209
216
print(f"Remaining files: {files}")
210
217
assert files.strip() == ""
211
218
''}
219
+
220
+
with subtest("Check that we can include the upload info in the payload"):
221
+
client.succeed("${makeTestDerivation} test2.nix")
222
+
test2_file = client.succeed("nix-build --no-out-link test2.nix")
223
+
client.succeed(f"attic push --force-preamble test {test2_file}")
224
+
client.succeed(f"nix-store --delete {test2_file}")
225
+
client.succeed(f"nix-store -r {test2_file}")
212
226
213
227
with subtest("Check that we can destroy the cache"):
214
228
client.succeed("attic cache info test")
+50
-10
server/src/api/v1/upload_path.rs
+50
-10
server/src/api/v1/upload_path.rs
···
11
11
extract::{BodyStream, Extension, Json},
12
12
http::HeaderMap,
13
13
};
14
-
use bytes::Bytes;
14
+
use bytes::{Bytes, BytesMut};
15
15
use chrono::Utc;
16
16
use digest::Output as DigestOutput;
17
17
use futures::future::join_all;
···
34
34
use crate::{RequestState, State};
35
35
use attic::api::v1::upload_path::{
36
36
UploadPathNarInfo, UploadPathResult, UploadPathResultKind, ATTIC_NAR_INFO,
37
+
ATTIC_NAR_INFO_PREAMBLE_SIZE,
37
38
};
38
39
use attic::hash::Hash;
39
-
use attic::stream::StreamHasher;
40
+
use attic::stream::{read_chunk_async, StreamHasher};
40
41
use attic::util::Finally;
41
42
42
43
use crate::chunking::chunk_stream;
···
52
53
///
53
54
/// TODO: Make this configurable
54
55
const CONCURRENT_CHUNK_UPLOADS: usize = 10;
56
+
57
+
/// The maximum size of the upload info JSON.
58
+
const MAX_NAR_INFO_SIZE: usize = 64 * 1024; // 64 KiB
55
59
56
60
type CompressorFn<C> = Box<dyn FnOnce(C) -> Box<dyn AsyncRead + Unpin + Send> + Send>;
57
61
···
116
120
headers: HeaderMap,
117
121
stream: BodyStream,
118
122
) -> ServerResult<Json<UploadPathResult>> {
123
+
let mut stream = StreamReader::new(
124
+
stream.map(|r| r.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))),
125
+
);
126
+
119
127
let upload_info: UploadPathNarInfo = {
120
-
let header = headers
121
-
.get(ATTIC_NAR_INFO)
122
-
.ok_or_else(|| ErrorKind::RequestError(anyhow!("X-Attic-Nar-Info must be set")))?;
128
+
if let Some(preamble_size_bytes) = headers.get(ATTIC_NAR_INFO_PREAMBLE_SIZE) {
129
+
// Read from the beginning of the PUT body
130
+
let preamble_size: usize = preamble_size_bytes
131
+
.to_str()
132
+
.map_err(|_| {
133
+
ErrorKind::RequestError(anyhow!(
134
+
"{} has invalid encoding",
135
+
ATTIC_NAR_INFO_PREAMBLE_SIZE
136
+
))
137
+
})?
138
+
.parse()
139
+
.map_err(|_| {
140
+
ErrorKind::RequestError(anyhow!(
141
+
"{} must be a valid unsigned integer",
142
+
ATTIC_NAR_INFO_PREAMBLE_SIZE
143
+
))
144
+
})?;
123
145
124
-
serde_json::from_slice(header.as_bytes()).map_err(ServerError::request_error)?
146
+
if preamble_size > MAX_NAR_INFO_SIZE {
147
+
return Err(ErrorKind::RequestError(anyhow!("Upload info is too large")).into());
148
+
}
149
+
150
+
let buf = BytesMut::with_capacity(preamble_size);
151
+
let preamble = read_chunk_async(&mut stream, buf)
152
+
.await
153
+
.map_err(|e| ErrorKind::RequestError(e.into()))?;
154
+
155
+
if preamble.len() != preamble_size {
156
+
return Err(ErrorKind::RequestError(anyhow!(
157
+
"Upload info doesn't match specified size"
158
+
))
159
+
.into());
160
+
}
161
+
162
+
serde_json::from_slice(&preamble).map_err(ServerError::request_error)?
163
+
} else if let Some(nar_info_bytes) = headers.get(ATTIC_NAR_INFO) {
164
+
// Read from X-Attic-Nar-Info header
165
+
serde_json::from_slice(nar_info_bytes.as_bytes()).map_err(ServerError::request_error)?
166
+
} else {
167
+
return Err(ErrorKind::RequestError(anyhow!("{} must be set", ATTIC_NAR_INFO)).into());
168
+
}
125
169
};
126
170
let cache_name = &upload_info.cache;
127
171
···
133
177
Ok(cache)
134
178
})
135
179
.await?;
136
-
137
-
let stream = StreamReader::new(
138
-
stream.map(|r| r.map_err(|e| io::Error::new(io::ErrorKind::Other, e.to_string()))),
139
-
);
140
180
141
181
let username = req_state.auth.username().map(str::to_string);
142
182