don't
fix: support 'tar.bz2' and 'tar.xz' archives and capture git-archive errors
Handle non-builtin formats for git-archive which are defined in the
lexicon. The commands used for `bzip2` and `xz` may be configured by
the knot operator.
Delay streaming the response to the client in an attempt to capture any
errors in the spawned process.
Signed-off-by: tjh <x@tjh.dev>
+110
-33
+33
-1
crates/knot/src/cli.rs
+33
-1
crates/knot/src/cli.rs
···
7
7
use url::Url;
8
8
9
9
pub fn parse() -> Arguments {
10
-
Arguments::parse()
10
+
let mut arguments = Arguments::parse();
11
+
12
+
if let Some("") = arguments.archive_bz2_command.as_deref() {
13
+
arguments.archive_bz2_command = None;
14
+
}
15
+
16
+
if let Some("") = arguments.archive_xz_command.as_deref() {
17
+
arguments.archive_xz_command = None;
18
+
}
19
+
20
+
arguments
11
21
}
12
22
13
23
#[derive(Clone, Debug, Parser)]
···
92
82
/// Seconds to retain a repository handle in cache.
93
83
#[arg(long, env = "KNOT_REPO_CACHE_LIVE", default_value_t = 600)]
94
84
pub repo_cache_live: u64,
85
+
86
+
/// Command to use to compress bzip2 archives.
87
+
#[arg(long, env = "KNOT_ARCHIVE_BZ2", default_value = find_command("bzip2").unwrap_or_default())]
88
+
pub archive_bz2_command: Option<String>,
89
+
90
+
/// Command to use to compress xz archives.
91
+
#[arg(long, env = "KNOT_ARCHIVE_XZ", default_value = find_command("xz").unwrap_or_default())]
92
+
pub archive_xz_command: Option<String>,
93
+
}
94
+
95
+
fn find_command(name: &str) -> Option<String> {
96
+
use std::process::Command;
97
+
98
+
let output = Command::new("which").arg(name).output().ok()?;
99
+
if !output.status.success() {
100
+
return None;
101
+
}
102
+
103
+
let full_path = String::from_utf8(output.stdout).ok()?;
104
+
Some(full_path.trim().to_string())
95
105
}
96
106
97
107
impl Arguments {
···
131
101
repo_cache_size,
132
102
repo_cache_idle,
133
103
repo_cache_live,
104
+
archive_bz2_command: _,
105
+
archive_xz_command: _,
134
106
} = self.clone();
135
107
136
108
// @TODO Validate?
+8
crates/knot/src/main.rs
+8
crates/knot/src/main.rs
···
80
80
}
81
81
)?);
82
82
83
+
if let Some(command) = &arguments.archive_bz2_command {
84
+
assert!(git_config_global_set("tar.tar.bz2.command", command)?);
85
+
}
86
+
87
+
if let Some(command) = &arguments.archive_xz_command {
88
+
assert!(git_config_global_set("tar.tar.xz.command", command)?);
89
+
}
90
+
83
91
let database = {
84
92
let pool = {
85
93
let connect_options = SqliteConnectOptions::new()
+61
-27
crates/knot/src/public/xrpc/sh_tangled/repo/impl_archive.rs
+61
-27
crates/knot/src/public/xrpc/sh_tangled/repo/impl_archive.rs
···
8
8
use axum::extract::State;
9
9
use axum::http::HeaderMap;
10
10
use axum::http::HeaderValue;
11
+
use axum::http::StatusCode;
11
12
use axum::response::IntoResponse;
12
13
use axum_extra::body::AsyncReadBody;
13
14
use git_service::util::SetOptionArg as _;
15
+
use lexicon::sh_tangled::repo::archive::Format;
14
16
use lexicon::sh_tangled::repo::archive::Input;
15
17
use std::process::Stdio;
18
+
use std::time::Duration;
16
19
17
20
pub const LXM: &str = "/sh.tangled.repo.archive";
21
+
22
+
const SPAWN_WAIT: Duration = Duration::from_millis(100);
18
23
19
24
#[tracing::instrument(target = "sh_tangled::repo::archive", skip(knot, repository), err)]
20
25
pub async fn handle(
···
39
34
.parse()
40
35
.expect("Repository extractor should have validated repo parameter");
41
36
42
-
let ResolvedRevspec { commit, immutable } = repository.resolve_revspec(&Some(rev.as_str()))?;
43
-
let rev = commit.id.to_string();
37
+
let (mut child, immutable, link) = {
38
+
let ResolvedRevspec { commit, immutable } =
39
+
repository.resolve_revspec(&Some(rev.as_str()))?;
44
40
45
-
let immutable_link = {
46
-
let base = format!(
47
-
"https://{}/xrpc/sh.tangled.repo.archive",
48
-
knot.instance_ident()
41
+
let rev = commit.id.to_string();
42
+
let immutable_link = immutable_link(
43
+
&format!("https://{}/xrpc{LXM}", knot.instance_ident()),
44
+
&repo,
45
+
&rev,
46
+
prefix.as_deref(),
47
+
format,
49
48
);
50
49
51
-
let mut url = url::Url::parse(&base).expect("Base URL should be valid");
52
-
{
53
-
let mut query = url.query_pairs_mut();
54
-
query.append_pair("repo", &repo);
55
-
query.append_pair("format", format.as_str());
56
-
query.append_pair("ref", &rev);
57
-
if let Some(prefix) = &prefix {
58
-
query.append_pair("prefix", prefix);
59
-
}
60
-
}
61
-
url
50
+
let mut command: tokio::process::Command = repository.git().into();
51
+
let child = command
52
+
.arg("archive")
53
+
.arg(format!("--format={format}"))
54
+
.option_arg(prefix.map(|prefix| format!("--prefix={prefix}/")))
55
+
.arg(&rev)
56
+
.stdout(Stdio::piped())
57
+
.stderr(Stdio::piped())
58
+
.spawn()?;
59
+
60
+
(child, immutable, immutable_link)
62
61
};
63
62
64
-
let mut command: tokio::process::Command = repository.git().into();
65
-
let mut child = command
66
-
.arg("archive")
67
-
.arg(format!("--format={format}"))
68
-
.option_arg(prefix.map(|prefix| format!("--prefix={prefix}/")))
69
-
.arg(&rev)
70
-
.stdout(Stdio::piped())
71
-
.spawn()?;
63
+
// Allow some time for the spawned command to run.
64
+
tokio::time::sleep(SPAWN_WAIT).await;
65
+
if let Ok(Some(exit_status)) = child.try_wait()
66
+
&& !exit_status.success()
67
+
{
68
+
tracing::error!(?exit_status, "failed to spawn git-archive");
69
+
let output = child.wait_with_output().await.map_err(errors::Internal)?;
70
+
let message = String::from_utf8_lossy(&output.stderr).trim().to_string();
71
+
return Err(XrpcError {
72
+
status: StatusCode::INTERNAL_SERVER_ERROR,
73
+
error: "ArchiveError".into(),
74
+
message: message.into(),
75
+
});
76
+
}
72
77
73
78
let stdout = child
74
79
.stdout
···
108
93
);
109
94
headers.insert(
110
95
header::LINK,
111
-
HeaderValue::from_str(&format!("<{immutable_link}>; rel=\"immutable\""))
112
-
.map_err(errors::Internal)?,
96
+
HeaderValue::from_str(&format!("<{link}>; rel=\"immutable\"")).map_err(errors::Internal)?,
113
97
);
114
98
115
99
Ok((headers, AsyncReadBody::new(stdout)).into_response())
100
+
}
101
+
102
+
fn immutable_link(
103
+
base: &str,
104
+
repo: &str,
105
+
revision: &str,
106
+
prefix: Option<&str>,
107
+
format: Format,
108
+
) -> url::Url {
109
+
let mut url = url::Url::parse(&base).expect("Base URL should be valid");
110
+
{
111
+
let mut query = url.query_pairs_mut();
112
+
query.append_pair("repo", &repo);
113
+
query.append_pair("format", format.as_str());
114
+
query.append_pair("ref", &revision);
115
+
if let Some(prefix) = &prefix {
116
+
query.append_pair("prefix", prefix);
117
+
}
118
+
}
119
+
url
116
120
}
+8
-5
crates/lexicon/src/sh_tangled/repo/archive.rs
+8
-5
crates/lexicon/src/sh_tangled/repo/archive.rs
···
22
22
pub prefix: Option<String>,
23
23
}
24
24
25
-
#[derive(Debug, Default, serde::Deserialize)]
25
+
#[derive(Clone, Copy, Debug, Default, serde::Deserialize)]
26
+
#[serde(rename_all = "lowercase")]
26
27
pub enum Format {
28
+
Tar,
27
29
#[default]
28
30
#[serde(rename = "tar.gz")]
29
31
TarGz,
···
33
31
TarBz2,
34
32
#[serde(rename = "tar.xz")]
35
33
TarXz,
36
-
#[serde(rename = "zip")]
37
34
Zip,
38
35
}
39
36
40
37
impl fmt::Display for Format {
41
-
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
38
+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
42
39
f.write_str(self.as_str())
43
40
}
44
41
}
45
42
46
43
impl Format {
47
-
#[must_use]
44
+
#[must_use]
48
45
pub const fn as_str(&self) -> &'static str {
49
46
match self {
47
+
Self::Tar => "tar",
50
48
Self::TarGz => "tar.gz",
51
49
Self::TarBz2 => "tar.bz2",
52
50
Self::TarXz => "tar.xz",
···
54
52
}
55
53
}
56
54
57
-
#[must_use]
55
+
#[must_use]
58
56
pub const fn as_content_type(&self) -> &'static str {
59
57
match self {
58
+
Self::Tar => "application/x-tar",
60
59
Self::TarGz => "application/gzip",
61
60
Self::TarBz2 => "application/x-bzip2",
62
61
Self::TarXz => "application/x-xz",