From 220462e562b9e0db79f0f7073139cafe72018084 Mon Sep 17 00:00:00 2001 From: YI Date: Mon, 6 Dec 2021 18:46:30 +0800 Subject: [PATCH 1/2] add argument `--counts` to run repeated tests --- src/bin/cargo/commands/bench.rs | 1 + src/bin/cargo/commands/test.rs | 2 ++ src/cargo/ops/cargo_test.rs | 25 ++++++++++++++----------- src/cargo/util/command_prelude.rs | 8 ++++++++ tests/testsuite/test.rs | 25 +++++++++++++++++++++++++ 5 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/bin/cargo/commands/bench.rs b/src/bin/cargo/commands/bench.rs index 3c6956c7f90..e0514420895 100644 --- a/src/bin/cargo/commands/bench.rs +++ b/src/bin/cargo/commands/bench.rs @@ -65,6 +65,7 @@ pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult { let ops = TestOptions { no_run: args.is_present("no-run"), no_fail_fast: args.is_present("no-fail-fast"), + counts: None, compile_opts, }; diff --git a/src/bin/cargo/commands/test.rs b/src/bin/cargo/commands/test.rs index 1be51e016b0..efafb9deffb 100644 --- a/src/bin/cargo/commands/test.rs +++ b/src/bin/cargo/commands/test.rs @@ -40,6 +40,7 @@ pub fn cli() -> App { .arg(opt("doc", "Test only this library's documentation")) .arg(opt("no-run", "Compile, but don't run tests")) .arg(opt("no-fail-fast", "Run all tests regardless of failure")) + .arg_counts() .arg_package_spec( "Package to run tests for", "Test all packages in the workspace", @@ -117,6 +118,7 @@ pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult { let ops = ops::TestOptions { no_run, no_fail_fast: args.is_present("no-fail-fast"), + counts: args.value_of_u32("counts")?, compile_opts, }; diff --git a/src/cargo/ops/cargo_test.rs b/src/cargo/ops/cargo_test.rs index c461c93a6a9..3e7e9fed0c4 100644 --- a/src/cargo/ops/cargo_test.rs +++ b/src/cargo/ops/cargo_test.rs @@ -11,6 +11,7 @@ pub struct TestOptions { pub compile_opts: ops::CompileOptions, pub no_run: bool, pub no_fail_fast: bool, + pub counts: Option, } pub fn run_tests( @@ -113,18 +114,20 @@ fn run_unit_tests( .shell() .verbose(|shell| shell.status("Running", &cmd))?; - let result = cmd.exec(); + for _ in 0..options.counts.unwrap_or(1) { + let result = cmd.exec(); - if let Err(e) = result { - let e = e.downcast::()?; - errors.push(( - unit.target.kind().clone(), - unit.target.name().to_string(), - unit.pkg.name().to_string(), - e, - )); - if !options.no_fail_fast { - break; + if let Err(e) = result { + let e = e.downcast::()?; + errors.push(( + unit.target.kind().clone(), + unit.target.name().to_string(), + unit.pkg.name().to_string(), + e, + )); + if !options.no_fail_fast { + break; + } } } } diff --git a/src/cargo/util/command_prelude.rs b/src/cargo/util/command_prelude.rs index 31ceb737cd4..7fcc49d95d0 100644 --- a/src/cargo/util/command_prelude.rs +++ b/src/cargo/util/command_prelude.rs @@ -66,6 +66,14 @@ pub trait AppExt: Sized { ) } + fn arg_counts(self) -> Self { + self._arg( + opt("counts", "Number of repeated runs, default to 1") + .short("c") + .value_name("N"), + ) + } + fn arg_jobs(self) -> Self { self._arg( opt("jobs", "Number of parallel jobs, defaults to # of CPUs") diff --git a/tests/testsuite/test.rs b/tests/testsuite/test.rs index eb66bc77c00..d539733d54a 100644 --- a/tests/testsuite/test.rs +++ b/tests/testsuite/test.rs @@ -4455,3 +4455,28 @@ fn test_workspaces_cwd() { .with_stdout_contains("test test_integration_deep_cwd ... ok") .run(); } + +#[cargo_test] +fn cargo_test_counts() { + let p = project() + .file("Cargo.toml", &basic_bin_manifest("foo")) + .file( + "src/main.rs", + r#" + fn main() {} + #[test] fn test_hello() {} + "#, + ) + .build(); + + p.cargo("test -c 4") + .with_stderr( + "\ +[COMPILING] foo v0.5.0 ([CWD]) +[FINISHED] test [unoptimized + debuginfo] target(s) in [..] +[RUNNING] [..] (target/debug/deps/foo-[..][EXE]) +", + ) + .with_stdout_contains_n("test test_hello ... ok", 4) + .run(); +} From 5db9637eb47ae59ad4d63576ab542a316a2334e9 Mon Sep 17 00:00:00 2001 From: YI Date: Mon, 6 Dec 2021 19:34:34 +0800 Subject: [PATCH 2/2] add argument `--counts` to run repeated benchmarks --- src/bin/cargo/commands/bench.rs | 3 ++- tests/testsuite/bench.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/bin/cargo/commands/bench.rs b/src/bin/cargo/commands/bench.rs index e0514420895..22421fce7ac 100644 --- a/src/bin/cargo/commands/bench.rs +++ b/src/bin/cargo/commands/bench.rs @@ -46,6 +46,7 @@ pub fn cli() -> App { "no-fail-fast", "Run all benchmarks regardless of failure", )) + .arg_counts() .arg_unit_graph() .after_help("Run `cargo help bench` for more detailed information.\n") } @@ -65,7 +66,7 @@ pub fn exec(config: &mut Config, args: &ArgMatches<'_>) -> CliResult { let ops = TestOptions { no_run: args.is_present("no-run"), no_fail_fast: args.is_present("no-fail-fast"), - counts: None, + counts: args.value_of_u32("counts")?, compile_opts, }; diff --git a/tests/testsuite/bench.rs b/tests/testsuite/bench.rs index a3923dced96..d00fdbe5925 100644 --- a/tests/testsuite/bench.rs +++ b/tests/testsuite/bench.rs @@ -1756,3 +1756,34 @@ fn json_artifact_includes_executable_for_benchmark() { ) .run(); } + +#[cargo_test] +fn cargo_bench_counts() { + if !is_nightly() { + return; + } + + let p = project() + .file("Cargo.toml", &basic_bin_manifest("foo")) + .file( + "src/main.rs", + r#" + #![feature(test)] + #[cfg(test)] + extern crate test; + fn main() {} + #[bench] fn bench_hello(_b: &mut test::Bencher) {} + "#, + ) + .build(); + + p.cargo("bench --counts 4") + .with_stderr( + "\ +[COMPILING] foo v0.5.0 ([CWD]) +[FINISHED] bench [optimized] target(s) in [..] +[RUNNING] [..] (target/release/deps/foo-[..][EXE])", + ) + .with_stdout_contains_n("test bench_hello ... bench: [..]", 4) + .run(); +}