diff --git a/crates/karva/src/commands/test/mod.rs b/crates/karva/src/commands/test/mod.rs index 28f58a06..5c492674 100644 --- a/crates/karva/src/commands/test/mod.rs +++ b/crates/karva/src/commands/test/mod.rs @@ -9,7 +9,7 @@ use karva_cache::AggregatedResults; use karva_cli::{OutputFormat, TestCommand}; use karva_logging::{Printer, Stdout, set_colored_override, setup_tracing}; use karva_metadata::filter::FiltersetSet; -use karva_metadata::{ProjectMetadata, ProjectOptionsOverrides}; +use karva_metadata::{NoTestsMode, ProjectMetadata, ProjectOptionsOverrides}; use karva_project::Project; use karva_project::path::absolute; use karva_python_semantic::current_python_version; @@ -87,6 +87,37 @@ pub fn test(args: TestCommand) -> Result { durations, )?; + if no_tests_matched_filters(&result) { + let has_filters = !sub_command.filter_expressions.is_empty(); + match project.settings().test().no_tests { + NoTestsMode::Auto => { + if has_filters { + return Ok(ExitStatus::Success); + } + let mut stdout = printer.stream_for_failure_summary().lock(); + writeln!( + stdout, + "error: no tests matched the provided filters (use --no-tests=pass or --no-tests=warn)" + )?; + return Ok(ExitStatus::Failure); + } + NoTestsMode::Pass => return Ok(ExitStatus::Success), + NoTestsMode::Warn => { + let mut stdout = printer.stream_for_requested_summary().lock(); + writeln!(stdout, "warning: no tests matched the provided filters")?; + return Ok(ExitStatus::Success); + } + NoTestsMode::Fail => { + let mut stdout = printer.stream_for_failure_summary().lock(); + writeln!( + stdout, + "error: no tests matched the provided filters (use --no-tests=pass or --no-tests=warn)" + )?; + return Ok(ExitStatus::Failure); + } + } + } + if result.stats.is_success() && result.discovery_diagnostics.is_empty() { Ok(ExitStatus::Success) } else { @@ -94,6 +125,12 @@ pub fn test(args: TestCommand) -> Result { } } +fn no_tests_matched_filters(result: &AggregatedResults) -> bool { + result.stats.total() == 0 + && result.discovery_diagnostics.is_empty() + && result.diagnostics.is_empty() +} + /// Print test output: diagnostics, durations, and result summary. pub fn print_test_output( printer: Printer, diff --git a/crates/karva/tests/it/basic.rs b/crates/karva/tests/it/basic.rs index 2193647e..275b7587 100644 --- a/crates/karva/tests/it/basic.rs +++ b/crates/karva/tests/it/basic.rs @@ -39,11 +39,12 @@ fn test_empty_file() { let context = TestContext::with_file("test.py", ""); assert_cmd_snapshot!(context.command(), @" - success: true - exit_code: 0 + success: false + exit_code: 1 ----- stdout ----- ──────────── Summary [TIME] 0 tests run: 0 passed, 0 skipped + error: no tests matched the provided filters (use --no-tests=pass or --no-tests=warn) ----- stderr ----- "); @@ -54,11 +55,12 @@ fn test_empty_directory() { let context = TestContext::new(); assert_cmd_snapshot!(context.command(), @" - success: true - exit_code: 0 + success: false + exit_code: 1 ----- stdout ----- ──────────── Summary [TIME] 0 tests run: 0 passed, 0 skipped + error: no tests matched the provided filters (use --no-tests=pass or --no-tests=warn) ----- stderr ----- "); @@ -140,11 +142,12 @@ fn test_no_tests_found() { let context = TestContext::with_file("test_no_tests.py", r""); assert_cmd_snapshot!(context.command(), @" - success: true - exit_code: 0 + success: false + exit_code: 1 ----- stdout ----- ──────────── Summary [TIME] 0 tests run: 0 passed, 0 skipped + error: no tests matched the provided filters (use --no-tests=pass or --no-tests=warn) ----- stderr ----- "); diff --git a/crates/karva/tests/it/filterset.rs b/crates/karva/tests/it/filterset.rs index 30a7b323..4d534d76 100644 --- a/crates/karva/tests/it/filterset.rs +++ b/crates/karva/tests/it/filterset.rs @@ -10,6 +10,11 @@ def test_beta(): assert True "; +const NO_TESTS: &str = r" +def helper(): + pass +"; + #[test] fn filterset_test_substring() { let context = TestContext::with_file("test.py", TWO_TESTS); @@ -74,20 +79,69 @@ fn filterset_test_multiple_flags_or_semantics() { #[test] fn filterset_test_no_matches() { - let context = TestContext::with_file("test.py", TWO_TESTS); - assert_cmd_snapshot!(context.command_no_parallel().arg("-E").arg("test(~nonexistent)"), @" + let context = TestContext::with_file("test.py", NO_TESTS); + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: false + exit_code: 1 + ----- stdout ----- + ──────────── + Summary [TIME] 0 tests run: 0 passed, 0 skipped + error: no tests matched the provided filters (use --no-tests=pass or --no-tests=warn) + + ----- stderr ----- + "); +} + +#[test] +fn filterset_test_no_matches_auto_with_filter() { + let context = TestContext::with_file("test.py", NO_TESTS); + assert_cmd_snapshot!( + context.command_no_parallel().arg("-E").arg("test(~helper)"), + @" success: true exit_code: 0 ----- stdout ----- - Starting 2 tests across 1 worker - SKIP [TIME] test::test_alpha - SKIP [TIME] test::test_beta + ──────────── + Summary [TIME] 0 tests run: 0 passed, 0 skipped + + ----- stderr ----- + " + ); +} +#[test] +fn filterset_test_no_matches_pass() { + let context = TestContext::with_file("test.py", NO_TESTS); + assert_cmd_snapshot!( + context.command_no_parallel().arg("--no-tests").arg("pass"), + @" + success: true + exit_code: 0 + ----- stdout ----- ──────────── - Summary [TIME] 2 tests run: 0 passed, 2 skipped + Summary [TIME] 0 tests run: 0 passed, 0 skipped ----- stderr ----- - "); + " + ); +} + +#[test] +fn filterset_test_no_matches_warn() { + let context = TestContext::with_file("test.py", NO_TESTS); + assert_cmd_snapshot!( + context.command_no_parallel().arg("--no-tests").arg("warn"), + @" + success: true + exit_code: 0 + ----- stdout ----- + ──────────── + Summary [TIME] 0 tests run: 0 passed, 0 skipped + warning: no tests matched the provided filters + + ----- stderr ----- + " + ); } #[test] diff --git a/crates/karva_cli/src/lib.rs b/crates/karva_cli/src/lib.rs index 0cedb6f8..b045fc53 100644 --- a/crates/karva_cli/src/lib.rs +++ b/crates/karva_cli/src/lib.rs @@ -5,7 +5,9 @@ use clap::Parser; use clap::builder::Styles; use clap::builder::styling::{AnsiColor, Effects}; use karva_logging::{TerminalColor, VerbosityLevel}; -use karva_metadata::{MaxFail, Options, RunIgnoredMode, SrcOptions, TerminalOptions, TestOptions}; +use karva_metadata::{ + MaxFail, NoTestsMode, Options, RunIgnoredMode, SrcOptions, TerminalOptions, TestOptions, +}; use ruff_db::diagnostic::DiagnosticFormat; const STYLES: Styles = Styles::styled() @@ -207,6 +209,19 @@ pub struct SubTestCommand { #[clap(short = 'E', long = "filter", help_heading = "Filter options")] pub filter_expressions: Vec, + /// Behavior when no tests are found to run [default: auto] + /// + /// `auto` fails if no filter expressions were given, and passes silently + /// if filters were given (the filter may legitimately match nothing on + /// some platforms or configurations). + #[arg( + long, + value_name = "ACTION", + env = "KARVA_NO_TESTS", + help_heading = "Filter options" + )] + pub no_tests: Option, + /// Run ignored tests. #[arg(long, help_heading = "Filter options")] pub run_ignored: Option, @@ -370,6 +385,7 @@ impl SubTestCommand { max_fail, try_import_fixtures: self.try_import_fixtures, retry: self.retry, + no_tests: self.no_tests.map(Into::into), }), } } @@ -408,3 +424,42 @@ impl From for RunIgnoredMode { } } } + +/// Behavior when no tests match filters. +#[derive(Copy, Clone, Hash, Debug, PartialEq, Eq, clap::ValueEnum)] +pub enum NoTests { + /// Automatically determine behavior: fail if no filter expressions were + /// given, pass silently if filters were given. + Auto, + + /// Silently exit with code 0. + Pass, + + /// Produce a warning and exit with code 0. + Warn, + + /// Produce an error message and exit with a non-zero code. + Fail, +} + +impl NoTests { + pub fn as_str(self) -> &'static str { + match self { + Self::Auto => "auto", + Self::Pass => "pass", + Self::Warn => "warn", + Self::Fail => "fail", + } + } +} + +impl From for NoTestsMode { + fn from(value: NoTests) -> Self { + match value { + NoTests::Auto => Self::Auto, + NoTests::Pass => Self::Pass, + NoTests::Warn => Self::Warn, + NoTests::Fail => Self::Fail, + } + } +} diff --git a/crates/karva_metadata/src/lib.rs b/crates/karva_metadata/src/lib.rs index 16b34f5a..fd706fba 100644 --- a/crates/karva_metadata/src/lib.rs +++ b/crates/karva_metadata/src/lib.rs @@ -14,7 +14,7 @@ pub use options::{ Options, OutputFormat, ProjectOptionsOverrides, SrcOptions, TerminalOptions, TestOptions, }; pub use pyproject::{PyProject, PyProjectError}; -pub use settings::{ProjectSettings, RunIgnoredMode}; +pub use settings::{NoTestsMode, ProjectSettings, RunIgnoredMode}; use crate::options::KarvaTomlError; diff --git a/crates/karva_metadata/src/options.rs b/crates/karva_metadata/src/options.rs index bec623a9..e941fc53 100644 --- a/crates/karva_metadata/src/options.rs +++ b/crates/karva_metadata/src/options.rs @@ -8,7 +8,7 @@ use thiserror::Error; use crate::filter::FiltersetSet; use crate::max_fail::MaxFail; use crate::settings::{ - ProjectSettings, RunIgnoredMode, SrcSettings, TerminalSettings, TestSettings, + NoTestsMode, ProjectSettings, RunIgnoredMode, SrcSettings, TerminalSettings, TestSettings, }; #[derive( @@ -215,6 +215,21 @@ pub struct TestOptions { "# )] pub retry: Option, + + /// Configures behavior when no tests are found to run. + /// + /// `auto` (the default) fails when no filter expressions were given, and + /// passes silently when filters were given. Use `fail` to always fail, + /// `warn` to always warn, or `pass` to always succeed silently. + #[serde(skip_serializing_if = "Option::is_none")] + #[option( + default = r#"auto"#, + value_type = "auto | pass | warn | fail", + example = r#" + no-tests = "warn" + "# + )] + pub no_tests: Option, } impl TestOptions { @@ -234,6 +249,7 @@ impl TestOptions { retry: self.retry.unwrap_or_default(), filter: FiltersetSet::default(), run_ignored: RunIgnoredMode::default(), + no_tests: self.no_tests.unwrap_or_default(), } } } @@ -364,7 +380,7 @@ nonsense = 42 | 4 | nonsense = 42 | ^^^^^^^^ - unknown field `nonsense`, expected one of `test-function-prefix`, `fail-fast`, `max-fail`, `try-import-fixtures`, `retry` + unknown field `nonsense`, expected one of `test-function-prefix`, `fail-fast`, `max-fail`, `try-import-fixtures`, `retry`, `no-tests` " ); } @@ -444,6 +460,7 @@ max-fail = 0 retry: Some( 5, ), + no_tests: None, } "#); } @@ -470,6 +487,7 @@ max-fail = 0 retry: Some( 3, ), + no_tests: None, } "#); } @@ -531,6 +549,7 @@ max-fail = 0 retry: Some( 2, ), + no_tests: None, }, ) "#); diff --git a/crates/karva_metadata/src/settings.rs b/crates/karva_metadata/src/settings.rs index d6c726d8..ae4b2cb0 100644 --- a/crates/karva_metadata/src/settings.rs +++ b/crates/karva_metadata/src/settings.rs @@ -1,3 +1,6 @@ +use karva_combine::Combine; +use serde::{Deserialize, Serialize}; + use crate::filter::FiltersetSet; use crate::max_fail::MaxFail; use crate::options::OutputFormat; @@ -10,6 +13,26 @@ pub enum RunIgnoredMode { All, } +#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub enum NoTestsMode { + #[default] + Auto, + Pass, + Warn, + Fail, +} + +impl Combine for NoTestsMode { + #[inline(always)] + fn combine_with(&mut self, _other: Self) {} + + #[inline] + fn combine(self, _other: Self) -> Self { + self + } +} + #[derive(Default, Debug, Clone)] pub struct ProjectSettings { pub(crate) terminal: TerminalSettings, @@ -63,4 +86,5 @@ pub struct TestSettings { pub retry: u32, pub filter: FiltersetSet, pub run_ignored: RunIgnoredMode, + pub no_tests: NoTestsMode, } diff --git a/docs/cli.md b/docs/cli.md index 0c5e00ad..59c6f03c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -65,7 +65,15 @@ karva test [OPTIONS] [PATH]...
--no-ignore

When set, .gitignore files will not be respected

--no-parallel

Disable parallel execution (equivalent to --num-workers 1)

--no-progress

When set, we will not show individual test case results during execution

-
--num-workers, -n num-workers

Number of parallel workers (default: number of CPU cores)

+
--no-tests action

Behavior when no tests are found to run [default: auto]

+

auto fails if no filter expressions were given, and passes silently if filters were given (the filter may legitimately match nothing on some platforms or configurations).

+

May also be set with the KARVA_NO_TESTS environment variable.

Possible values:

+
    +
  • auto: Automatically determine behavior: fail if no filter expressions were given, pass silently if filters were given
    • +
  • pass: Silently exit with code 0
    • +
  • warn: Produce a warning and exit with code 0
    • +
  • fail: Produce an error message and exit with a non-zero code
    • +
--num-workers, -n num-workers

Number of parallel workers (default: number of CPU cores)

--output-format output-format

The format to use for printing diagnostic messages

Possible values:

    diff --git a/docs/configuration.md b/docs/configuration.md index 8ac92acc..5f91d4ff 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -133,6 +133,27 @@ max-fail = 3 --- +### `no-tests` + +Configures behavior when no tests are found to run. + +`auto` (the default) fails when no filter expressions were given, and +passes silently when filters were given. Use `fail` to always fail, +`warn` to always warn, or `pass` to always succeed silently. + +**Default value**: `auto` + +**Type**: `auto | pass | warn | fail` + +**Example usage** (`pyproject.toml`): + +```toml +[tool.karva.test] +no-tests = "warn" +``` + +--- + ### `retry` When set, we will retry failed tests up to this number of times.