Local runner for GitHub autograder
Way better outputting, support no comparison
+190
-87
+30
-14
.github/workflows/release.yml
+30
-14
.github/workflows/release.yml
···
1
-
2
1
name: Release
3
-
on:
2
+
on:
4
3
workflow_dispatch:
5
-
inputs:
6
-
tag:
7
-
description: Version Tag
8
-
type: string
9
-
4
+
10
5
permissions:
11
6
contents: write
7
+
id-token: write
8
+
attestations: write
12
9
13
10
jobs:
14
11
release:
···
23
20
with:
24
21
toolchain: stable
25
22
26
-
- name: Build
23
+
- name: Get Metadata
24
+
id: metadata
27
25
run: |
28
-
cargo build --release
26
+
echo "meta=$(cargo metadata --no-deps --frozen --format-version 1)" >> $GITHUB_OUTPUT
27
+
28
+
- name: Check Tag
29
+
id: check-tag
30
+
run: |
31
+
echo "exists=$(git ls-remote --exit-code --tags origin ${{ env.TAG }} >/dev/null 2>&1 && echo true || echo false)" >> $GITHUB_OUTPUT
32
+
echo "tag=${{ env.TAG }}" >> $GITHUB_OUTPUT
33
+
env:
34
+
TAG: "v${{fromJson(steps.metadata.outputs.meta).packages[0].version}}"
35
+
36
+
- name: Block if There's a Release
37
+
if: ${{ steps.check-tag.outputs.exists != 'false' }}
38
+
run: echo "::error file=Cargo.toml,title=Refusing to Release::Tag ${{ steps.check-tag.outputs.tag }} already exists" && exit 1
39
+
40
+
- name: Build
41
+
run: |
42
+
cargo build --frozen --release
43
+
44
+
- name: Generate artifact attestation
45
+
uses: actions/attest-build-provenance@v2
46
+
with:
47
+
subject-path: 'target/release/gh-grader-preview'
29
48
30
49
- name: Upload Binary Artifact
31
50
uses: actions/upload-artifact@v4
···
36
55
- name: Create Release
37
56
uses: softprops/action-gh-release@v2
38
57
with:
39
-
name: GH Grader Preview ${{ inputs.tag }}
40
-
tag_name: v${{ inputs.tag }}
58
+
name: GH Grader Preview ${{ steps.check-tag.outputs.tag }}
59
+
tag_name: ${{ steps.check-tag.outputs.tag }}
41
60
generate_release_notes: false
42
61
draft: true
43
62
files: |
44
63
target/release/gh-grader-preview
45
-
46
-
47
-
+1
-1
Cargo.lock
+1
-1
Cargo.lock
+1
-3
Cargo.toml
+1
-3
Cargo.toml
+34
-10
src/grader.rs
+34
-10
src/grader.rs
···
1
+
use std::fmt::Display;
2
+
1
3
use anyhow::Result;
2
4
use serde::Deserialize;
3
5
···
11
13
#[serde(rename_all = "snake_case")]
12
14
pub enum ComparisonType {
13
15
#[default]
16
+
Pass,
14
17
Included,
15
18
Excluded,
16
19
Exact,
17
20
Regex,
18
21
}
19
22
23
+
impl Display for ComparisonType {
24
+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
25
+
let s = match self {
26
+
ComparisonType::Included => "include this string",
27
+
ComparisonType::Excluded => "not include this string",
28
+
ComparisonType::Exact => "exactly match this string",
29
+
ComparisonType::Regex => "match against this regular expression",
30
+
ComparisonType::Pass => "can be anything",
31
+
};
32
+
f.write_str(s)
33
+
}
34
+
}
35
+
20
36
#[derive(Deserialize)]
21
37
pub struct TestCase {
22
38
pub name: String,
23
-
pub setup: String,
39
+
pub setup: Option<String>,
24
40
pub run: String,
25
41
#[serde(default)]
26
42
pub input: String,
27
-
pub output: String,
43
+
pub output: Option<String>,
44
+
#[serde(default)]
28
45
pub comparison: ComparisonType,
29
46
pub timeout: u32,
30
47
#[allow(unused)]
···
56
73
57
74
impl TestCase {
58
75
pub fn check_output(&self, output: String) -> Result<bool> {
59
-
match self.comparison {
60
-
ComparisonType::Included => Ok(output.contains(&self.output)),
61
-
ComparisonType::Excluded => Ok(!output.contains(&self.output)),
62
-
ComparisonType::Exact => Ok(output.trim() == self.output.trim()),
63
-
ComparisonType::Regex => {
64
-
let re = regex::Regex::new(&self.output)?;
65
-
Ok(re.is_match(&output))
76
+
if let Some(ref expected_output) = self.output {
77
+
match self.comparison {
78
+
ComparisonType::Included => Ok(output.contains(expected_output)),
79
+
ComparisonType::Excluded => Ok(!output.contains(expected_output)),
80
+
ComparisonType::Exact => Ok(output.trim() == expected_output.trim()),
81
+
ComparisonType::Regex => {
82
+
let re = regex::Regex::new(expected_output)?;
83
+
Ok(re.is_match(&output))
84
+
}
85
+
ComparisonType::Pass => Ok(true),
66
86
}
87
+
} else {
88
+
Ok(true)
67
89
}
68
90
}
69
91
70
92
pub fn run(&self) -> Result<(bool, TestResult)> {
71
-
runner::setup_phase(&self.setup)?;
93
+
if let Some(setup) = self.setup.as_ref() {
94
+
runner::setup_phase(setup)?;
95
+
}
72
96
let res = runner::run_phase(&self.run, &self.input, self.timeout as u64)?;
73
97
let matches = self.check_output(res.stdout.clone())?;
74
98
Ok((matches, res))
+122
-57
src/main.rs
+122
-57
src/main.rs
···
12
12
use grader::AutoGraderData;
13
13
use indicatif::ProgressBar;
14
14
15
+
use crate::{grader::ComparisonType, runner::TestResult};
16
+
17
+
struct CmdOutput(String, String, Option<String>);
18
+
19
+
impl CmdOutput {
20
+
fn format_output(self, name: &str, comp: &ComparisonType) -> String {
21
+
let CmdOutput(stdout, stderr, expected) = self;
22
+
23
+
let heading = format!("┏━┻ Stdout of {name}");
24
+
let stdout = stdout.trim().replace("\n", "\n┃ ");
25
+
let mid = format!("┣━━ Stderr of {name}");
26
+
let stderr = stderr.trim().replace("\n", "\n┃ ");
27
+
let (mid2, expected) = if let Some(expected) = expected {
28
+
(
29
+
format!("┣━━ Stdout of {name} must {comp}"),
30
+
expected.trim().replace("\n", "\n┃ "),
31
+
)
32
+
} else {
33
+
(format!("┣━━ {name} has no output check"), String::new())
34
+
};
35
+
let foot = "┗━━ End of output".to_string();
36
+
37
+
format!("{heading}\n┃ {stdout}\n{mid}\n┃ {stderr}\n{mid2}\n┃ {expected}\n{foot}")
38
+
}
39
+
}
40
+
41
+
#[derive(Debug)]
42
+
enum TestOutput {
43
+
Passed,
44
+
LogicError,
45
+
NonZeroExit(Option<i32>),
46
+
RunnerError(anyhow::Error),
47
+
Skipped,
48
+
}
49
+
50
+
impl From<Result<(bool, TestResult)>> for TestOutput {
51
+
fn from(res: Result<(bool, TestResult)>) -> Self {
52
+
match res {
53
+
Ok((matched, TestResult { status: Ok(0), .. })) => {
54
+
if matched {
55
+
Self::Passed
56
+
} else {
57
+
Self::LogicError
58
+
}
59
+
}
60
+
Ok((_, TestResult { status, .. })) => Self::NonZeroExit(status.ok()),
61
+
Err(why) => Self::RunnerError(why),
62
+
}
63
+
}
64
+
}
65
+
66
+
impl TestOutput {
67
+
fn output(&self) -> (&str, String) {
68
+
match self {
69
+
TestOutput::Passed => ("✅", "Passed".to_string()),
70
+
TestOutput::LogicError => ("❌", "Did not match expected output".to_string()),
71
+
TestOutput::NonZeroExit(code) => (
72
+
"💥",
73
+
format!("Exited with code {}", code.unwrap_or(i32::MAX)),
74
+
),
75
+
TestOutput::RunnerError(error) => ("🙈", format!("Wasn't able to run: {error:?}")),
76
+
TestOutput::Skipped => ("〰️", "Was skipped".to_string()),
77
+
}
78
+
}
79
+
}
80
+
15
81
fn main() -> Result<()> {
16
82
let cli = Cli::parse();
17
83
···
22
88
println!("{}", meta::gen_completions(shell.parse().unwrap()));
23
89
return Ok(());
24
90
}
91
+
92
+
let target_test = cli.test.as_ref().map(|s| s.to_lowercase());
25
93
26
94
let grader_data = AutoGraderData::get(cli.file)?;
27
95
28
-
let test_len = grader_data.tests.len();
29
-
30
-
let amount_passed = grader_data
96
+
let outputs = grader_data
31
97
.tests
32
98
.into_iter()
33
99
.enumerate()
34
-
.filter(|(i, test)| {
100
+
.map(|(i, test)| {
101
+
// In verbose mode make the output a bit easier to follow
102
+
if cli.verbose {
103
+
println!();
104
+
}
35
105
let bar = ProgressBar::new_spinner();
36
106
bar.set_message(format!("Running {}", test.name));
37
107
bar.enable_steady_tick(Duration::from_millis(100));
38
-
if cli.skip.map(|skip| i < &skip).unwrap_or(false)
39
-
|| cli
40
-
.test
41
-
.as_ref()
42
-
.map(|target| target.to_lowercase() != test.name.to_lowercase())
43
-
.unwrap_or(false)
44
-
{
45
-
bar.finish_with_message(format!("〰️ Skipped {}", test.name));
46
-
return false;
108
+
109
+
let skip_number = cli.skip.map(|skip| i < skip).unwrap_or(false);
110
+
111
+
let skip_non_target = target_test
112
+
.as_ref()
113
+
.is_some_and(|s| *s != test.name.to_lowercase());
114
+
115
+
let (output, verbose_output) = if skip_number || skip_non_target {
116
+
(TestOutput::Skipped, None)
117
+
} else {
118
+
let result = test.run();
119
+
let verbose_output =
120
+
result
121
+
.as_ref()
122
+
.ok()
123
+
.map(|(_, TestResult { stdout, stderr, .. })| {
124
+
CmdOutput(stdout.clone(), stderr.clone(), test.output.clone())
125
+
});
126
+
(result.into(), verbose_output)
127
+
};
128
+
129
+
let (emoji, msg) = output.output();
130
+
131
+
bar.finish_with_message(format!("{emoji} {} - {msg}", test.name));
132
+
133
+
if let (Some(verbose_output), true) = (verbose_output, cli.verbose) {
134
+
println!(
135
+
"{}",
136
+
verbose_output.format_output(&test.name, &test.comparison)
137
+
);
47
138
}
48
-
match test.run() {
49
-
Ok((matched, result)) => {
50
-
if cli.verbose {
51
-
println!(
52
-
"Stdout of {}:\n{}\n\nStderr of {}:\n{}",
53
-
test.name, result.stdout, test.name, result.stderr
54
-
);
55
-
}
56
-
match result.status {
57
-
Ok(code) => {
58
-
if code != 0 {
59
-
bar.finish_with_message(format!(
60
-
"⚠️ {} Exited with code {code}",
61
-
test.name
62
-
))
63
-
}
64
-
if matched {
65
-
bar.finish_with_message(format!("✅ {} passed!", test.name));
66
-
true
67
-
} else {
68
-
bar.finish_with_message(format!(
69
-
"❌ {} did not give the correct output",
70
-
test.name
71
-
));
72
-
false
73
-
}
74
-
}
75
-
Err(why) => {
76
-
bar.finish_with_message(format!(
77
-
"❌ Failed to run {}: {why:?}",
78
-
test.name
79
-
));
80
-
false
81
-
}
82
-
}
83
-
}
84
-
Err(why) => {
85
-
bar.finish_with_message(format!("❌ Failed to run {}: {why:?}", test.name));
86
-
false
87
-
}
88
-
}
139
+
140
+
output
89
141
})
142
+
.collect::<Vec<_>>();
143
+
144
+
let total_tests = outputs.len();
145
+
let total_not_skipped = outputs
146
+
.iter()
147
+
.filter(|o| !matches!(o, TestOutput::Skipped))
148
+
.count();
149
+
let total_passed = outputs
150
+
.iter()
151
+
.filter(|o| matches!(o, TestOutput::Passed))
90
152
.count();
91
153
92
-
println!("Passed {amount_passed}/{test_len} tests");
93
-
if cli.skip.is_some() || cli.test.is_some() {
94
-
println!("(Some were skipped)");
154
+
let percent_passed = total_passed as f32 / total_not_skipped as f32 * 100_f32;
155
+
156
+
println!("\n== TEST SUMMARY ==");
157
+
println!("{total_passed} / {total_not_skipped} Tests Passed ({percent_passed:.2}%)");
158
+
if total_tests != total_not_skipped {
159
+
println!("({} Tests Were Skipped)", total_tests - total_not_skipped);
95
160
}
96
161
97
162
Ok(())