Skip to content

Commit 8441388

Browse files
test(runner): honor family nextest config and add --test-threads
The runner passes its openshell profile as a nextest tool config and the bundle carries tests/suites/<family>/.config/nextest.toml into the remapped workspace, so a family can set per-test timeouts and test groups, or override the profile. --test-threads defaults to 1, which keeps the serial --no-capture run. Refs #5275 Signed-off-by: Matthew Grossman <[email protected]>
1 parent 283b9ff commit 8441388

4 files changed

Lines changed: 146 additions & 36 deletions

File tree

‎TESTING.md‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,24 @@ need different `TMACHINE_SSH_PORT` values (default 2222). The installer
247247
inputs, such as the Debian package and the images, come from `artifacts/`,
248248
built by `nix run .#build-artifacts` or downloaded by CI.
249249

250+
The runner's nextest profile, `openshell`, runs tests one at a time and
251+
fails a test that runs longer than two minutes (`slow-timeout` 30 s × 4). A
252+
family that needs other settings puts them in
253+
`tests/suites/<family>/.config/nextest.toml`, which takes priority over the
254+
runner's profile:
255+
256+
- `[[profile.default.overrides]]` for some tests, such as a longer
257+
`slow-timeout` or a `test-group` from `[test-groups]` that keeps tests from
258+
running at the same time. These also apply when you run the suite with
259+
`cargo nextest run`.
260+
- `[profile.openshell]` for every test of the family in the runner, such as
261+
its `slow-timeout`.
262+
263+
`--test-threads <N>` runs N tests at once. Tests share the gateway and the
264+
guest, so it defaults to 1; raise it only for tests that tolerate that, and
265+
put the ones that don't in a test group. A serial run streams test output into
266+
`nextest.log`; a parallel run shows a test's output only when it fails.
267+
250268
A flaky or broken test is marked `#[ignore = "<issue link>"]`, which takes it
251269
out of every environment until it's fixed.
252270

‎tests/runner/src/bundle.rs‎

Lines changed: 53 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,16 @@
44
//! Building a family's test bundle for the guest, on the host.
55
//!
66
//! A bundle is a directory holding a nextest archive built for the guest target,
7-
//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), a
8-
//! `nextest` config with the runner's `JUnit` profile, and `testkit-ready`,
9-
//! which waits for the gateway before the tests run. The guest needs only the
10-
//! `cargo-nextest` binary to run it.
7+
//! the family workspace's `Cargo.toml` files (for `--workspace-remap`), the
8+
//! runner's nextest profile, and `testkit-ready`, which waits for the gateway
9+
//! before the tests run. The guest needs only the `cargo-nextest` binary to run
10+
//! it.
11+
//!
12+
//! The runner's profile is a nextest tool config, below the repository config
13+
//! in priority. A family that needs other settings, such as longer timeouts for
14+
//! some tests or a test group that runs serially, puts them in
15+
//! `tests/suites/<family>/.config/nextest.toml`; the bundle carries it to the
16+
//! same place in the remapped workspace, where nextest reads it.
1117
1218
use std::path::{Path, PathBuf};
1319
use std::process::Command;
@@ -22,9 +28,20 @@ pub const PROFILE: &str = "openshell";
2228
/// The readiness check every bundle carries, at the bundle root.
2329
pub const READY_BINARY: &str = "testkit-ready";
2430

31+
/// The runner's nextest config, at the bundle root. It is passed as the tool
32+
/// config of [`TOOL_NAME`].
33+
pub const TOOL_CONFIG: &str = "runner-nextest.toml";
34+
35+
/// The tool name nextest gives the runner's config.
36+
pub const TOOL_NAME: &str = "openshell-test-runner";
37+
38+
/// A family's nextest config, relative to its workspace and the bundle root.
39+
const FAMILY_CONFIG: &str = ".config/nextest.toml";
40+
2541
/// Nextest flags a test as slow after the period and terminates it after
2642
/// `terminate-after` periods, so a hung test fails after two minutes instead
27-
/// of holding the lane until the job times out.
43+
/// of holding the lane until the job times out. A family's own config takes
44+
/// priority over this.
2845
const NEXTEST_CONFIG: &str = r#"# Written by the OpenShell test runner.
2946
[profile.openshell]
3047
fail-fast = false
@@ -134,12 +151,26 @@ pub fn build(workspace: &Path, packages: &[String], target: &str, ready: &Path)
134151
run(&mut command).with_context(|| format!("build the {} test archive", workspace.display()))?;
135152

136153
copy_manifests(workspace, stage)?;
137-
std::fs::write(stage.join("nextest.toml"), NEXTEST_CONFIG)?;
154+
copy_family_config(workspace, stage)?;
155+
std::fs::write(stage.join(TOOL_CONFIG), NEXTEST_CONFIG)?;
138156
std::fs::copy(ready, stage.join(READY_BINARY))
139157
.with_context(|| format!("copy {}", ready.display()))?;
140158
Ok(Bundle(dir))
141159
}
142160

161+
/// Copy the family's nextest config, if it has one, to the same place in
162+
/// `stage`.
163+
fn copy_family_config(workspace: &Path, stage: &Path) -> Result<()> {
164+
let config = workspace.join(FAMILY_CONFIG);
165+
if !config.is_file() {
166+
return Ok(());
167+
}
168+
let dest = stage.join(FAMILY_CONFIG);
169+
std::fs::create_dir_all(dest.parent().unwrap_or(stage))?;
170+
std::fs::copy(&config, &dest).with_context(|| format!("copy {}", config.display()))?;
171+
Ok(())
172+
}
173+
143174
/// Copy every `Cargo.toml` under `workspace` into `stage`, keeping relative
144175
/// paths and skipping build output.
145176
fn copy_manifests(workspace: &Path, stage: &Path) -> Result<()> {
@@ -188,4 +219,20 @@ mod tests {
188219
);
189220
assert_eq!(built_executable(messages, "other"), None);
190221
}
222+
223+
#[test]
224+
fn family_config_lands_where_nextest_reads_it() {
225+
let workspace = tempfile::tempdir().unwrap();
226+
let stage = tempfile::tempdir().unwrap();
227+
copy_family_config(workspace.path(), stage.path()).unwrap();
228+
assert!(!stage.path().join(".config").exists());
229+
230+
std::fs::create_dir(workspace.path().join(".config")).unwrap();
231+
std::fs::write(workspace.path().join(FAMILY_CONFIG), "[test-groups]\n").unwrap();
232+
copy_family_config(workspace.path(), stage.path()).unwrap();
233+
assert_eq!(
234+
std::fs::read_to_string(stage.path().join(".config/nextest.toml")).unwrap(),
235+
"[test-groups]\n"
236+
);
237+
}
191238
}

‎tests/runner/src/execute.rs‎

Lines changed: 68 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
//! [`crate::test_env`]).
1313
1414
use std::fmt;
15+
use std::num::NonZeroUsize;
1516
use std::path::Path;
1617
use std::process::{ExitCode, ExitStatus, Output};
1718
use std::time::Duration;
@@ -20,7 +21,7 @@ use anyhow::{Context, Result, ensure};
2021
use tmachine::Session;
2122
use tmachine::config::Config;
2223

23-
use crate::bundle::{Bundle, PROFILE, READY_BINARY};
24+
use crate::bundle::{Bundle, PROFILE, READY_BINARY, TOOL_CONFIG, TOOL_NAME};
2425
use crate::test_env;
2526

2627
const GUEST_DIR: &str = "/var/tmp/openshell-tests";
@@ -114,6 +115,8 @@ pub struct Run<'a> {
114115
pub bundle: &'a Bundle,
115116
/// A nextest filterset (`-E`) narrowing the tests in the bundle.
116117
pub filter: Option<&'a str>,
118+
/// How many tests run at once.
119+
pub test_threads: NonZeroUsize,
117120
/// Where `junit.xml`, `nextest.log` and, after a failure, the guest's
118121
/// `diagnostics/` go.
119122
pub results: &'a Path,
@@ -282,37 +285,10 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
282285
)?;
283286
}
284287

285-
let archive = format!("{guest}/tests.tar.zst");
286-
let extract = format!("{guest}/extract");
287-
let config = format!("{guest}/nextest.toml");
288288
let mut nextest = vec!["env"];
289289
nextest.extend(env.iter().map(String::as_str));
290-
nextest.extend([
291-
"cargo-nextest",
292-
"nextest",
293-
"run",
294-
"--archive-file",
295-
&archive,
296-
"--workspace-remap",
297-
guest,
298-
"--extract-to",
299-
&extract,
300-
"--extract-overwrite",
301-
"--config-file",
302-
&config,
303-
"--profile",
304-
PROFILE,
305-
"--no-capture",
306-
// Tests share one gateway and the guest has 4 GiB; run them serially
307-
// whatever --no-capture implies.
308-
"--test-threads",
309-
"1",
310-
"--no-tests",
311-
"fail",
312-
]);
313-
if let Some(filter) = run.filter {
314-
nextest.extend(["-E", filter]);
315-
}
290+
let args = nextest_args(guest, run.test_threads, run.filter);
291+
nextest.extend(args.iter().map(String::as_str));
316292
// nextest writes its output to a file in the guest, so a run that times
317293
// out or loses SSH still leaves a log for `collect_results`. The guest
318294
// shell exports the published variables itself, so their values, which
@@ -331,6 +307,47 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
331307
))
332308
}
333309

310+
/// The `cargo-nextest` command line that runs the bundle unpacked at `guest`.
311+
///
312+
/// The runner's profile is a tool config, so the family's
313+
/// `.config/nextest.toml` in the remapped workspace overrides it. Tests share
314+
/// one gateway and the guest has 4 GiB, so they run one at a time unless the
315+
/// run asks for more threads. A serial run uses `--no-capture`, which runs
316+
/// tests one at a time and streams their output into the log. A parallel run
317+
/// can't, so nextest captures each test's output and shows it when the test
318+
/// fails.
319+
fn nextest_args(guest: &str, test_threads: NonZeroUsize, filter: Option<&str>) -> Vec<String> {
320+
let mut args: Vec<String> = [
321+
"cargo-nextest",
322+
"nextest",
323+
"run",
324+
"--archive-file",
325+
&format!("{guest}/tests.tar.zst"),
326+
"--workspace-remap",
327+
guest,
328+
"--extract-to",
329+
&format!("{guest}/extract"),
330+
"--extract-overwrite",
331+
"--tool-config-file",
332+
&format!("{TOOL_NAME}:{guest}/{TOOL_CONFIG}"),
333+
"--profile",
334+
PROFILE,
335+
"--no-tests",
336+
"fail",
337+
]
338+
.map(String::from)
339+
.into();
340+
if test_threads.get() == 1 {
341+
args.push("--no-capture".to_string());
342+
} else {
343+
args.extend(["--test-threads".to_string(), test_threads.to_string()]);
344+
}
345+
if let Some(filter) = filter {
346+
args.extend(["-E".to_string(), filter.to_string()]);
347+
}
348+
args
349+
}
350+
334351
/// Copy the nextest log and the JUnit report out of the guest, whether or not
335352
/// the run finished. Either may be missing, for example when the gateway never
336353
/// became ready.
@@ -525,6 +542,27 @@ mod tests {
525542
}
526543
}
527544

545+
#[test]
546+
fn nextest_runs_serially_with_output_unless_asked_for_threads() {
547+
let one = NonZeroUsize::MIN;
548+
let serial = nextest_args("/g", one, None);
549+
assert!(serial.contains(&"--no-capture".to_string()));
550+
// nextest ignores --test-threads with --no-capture, and warns.
551+
assert!(!serial.contains(&"--test-threads".to_string()));
552+
assert!(serial.windows(2).any(|w| w
553+
== [
554+
"--tool-config-file",
555+
"openshell-test-runner:/g/runner-nextest.toml"
556+
]));
557+
assert!(!serial.contains(&"--config-file".to_string()));
558+
assert!(!serial.contains(&"-E".to_string()));
559+
560+
let parallel = nextest_args("/g", NonZeroUsize::new(4).unwrap(), Some("test(a)"));
561+
assert!(!parallel.contains(&"--no-capture".to_string()));
562+
assert!(parallel.windows(2).any(|w| w == ["--test-threads", "4"]));
563+
assert!(parallel.ends_with(&["-E".to_string(), "test(a)".to_string()]));
564+
}
565+
528566
#[test]
529567
fn exit_code_separates_test_failures_from_errors() {
530568
assert_eq!(Status::Passed.exit_code(), ExitCode::SUCCESS);

‎tests/runner/src/main.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
//! provides the guest cross-compilation toolchain, QEMU, Ansible, and the
88
//! tmachine configuration.
99
10+
use std::num::NonZeroUsize;
1011
use std::path::{Path, PathBuf};
1112
use std::process::ExitCode;
1213

@@ -41,6 +42,11 @@ struct Cli {
4142
/// `binary(smoke)`.
4243
#[arg(long, value_name = "FILTERSET")]
4344
filter: Option<String>,
45+
/// How many tests run at once. Tests share the gateway and the guest, so
46+
/// raise it only for tests that tolerate that; nextest then captures each
47+
/// test's output and shows it when the test fails.
48+
#[arg(long, default_value = "1", value_name = "N")]
49+
test_threads: NonZeroUsize,
4450
/// The gateway name the installer registers with the CLI. The packaged
4551
/// installers register `openshell`.
4652
#[arg(long, default_value = "openshell")]
@@ -102,6 +108,7 @@ async fn run(cli: Cli) -> Result<ExitCode> {
102108
gateway: &cli.gateway,
103109
bundle: &bundle,
104110
filter: cli.filter.as_deref(),
111+
test_threads: cli.test_threads,
105112
results: &results,
106113
keep: cli.keep,
107114
},

0 commit comments

Comments
 (0)