diff --git a/crates/karva/tests/it/basic.rs b/crates/karva/tests/it/basic.rs index a75370ca..d06d5645 100644 --- a/crates/karva/tests/it/basic.rs +++ b/crates/karva/tests/it/basic.rs @@ -1920,3 +1920,467 @@ def test_2(): pass ----- stderr ----- "); } + +#[test] +fn test_color_never_strips_ansi() { + let context = TestContext::with_file("test.py", "def test_1(): pass"); + + assert_cmd_snapshot!(context.command_no_parallel().args(["--color", "never"]), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test::test_1 + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn test_color_invalid_value() { + let context = TestContext::with_file("test.py", "def test_1(): pass"); + + assert_cmd_snapshot!(context.command().args(["--color", "rainbow"]), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: invalid value 'rainbow' for '--color ' + [possible values: auto, always, never] + + For more information, try '--help'. + "); +} + +/// `--no-cache` disables reading duration history but the run should still succeed. +#[test] +fn test_no_cache_flag() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): pass +def test_2(): pass +", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("--no-cache"), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 2 tests across 1 worker + PASS [TIME] test::test_1 + PASS [TIME] test::test_2 + + ──────────── + Summary [TIME] 2 tests run: 2 passed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn test_no_progress_hides_per_test_lines() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): pass +def test_2(): pass +def test_3(): pass +", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("--no-progress"), @" + success: true + exit_code: 0 + ----- stdout ----- + + ──────────── + Summary [TIME] 3 tests run: 3 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// `--no-progress` still emits diagnostics for failing tests. +#[test] +fn test_no_progress_with_failure_shows_diagnostics() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): pass +def test_2(): assert False +", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("--no-progress"), @" + success: false + exit_code: 1 + ----- stdout ----- + + diagnostics: + + error[test-failure]: Test `test_2` failed + --> test.py:3:5 + | + 2 | def test_1(): pass + 3 | def test_2(): assert False + | ^^^^^^ + | + info: Test failed here + --> test.py:3:1 + | + 2 | def test_1(): pass + 3 | def test_2(): assert False + | ^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + + ──────────── + Summary [TIME] 2 tests run: 1 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} + +/// `--retry 0` is a no-op — failing tests still fail and are not re-run. +#[test] +fn test_retry_zero_is_noop() { + let context = TestContext::with_file( + "test.py", + r" +def test_fail(): assert False +", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("--retry").arg("0"), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 1 test across 1 worker + FAIL [TIME] test::test_fail + + diagnostics: + + error[test-failure]: Test `test_fail` failed + --> test.py:2:5 + | + 2 | def test_fail(): assert False + | ^^^^^^^^^ + | + info: Test failed here + --> test.py:2:1 + | + 2 | def test_fail(): assert False + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} + +/// A test that always fails exhausts retries and ends up reported as failed. +#[test] +fn test_retry_exhausts_on_always_failing_test() { + let context = TestContext::with_file( + "test.py", + r" +def test_always_fails(): assert False +", + ); + + assert_cmd_snapshot!(context.command_no_parallel().arg("--retry").arg("2"), @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 1 test across 1 worker + FAIL [TIME] test::test_always_fails + + diagnostics: + + error[test-failure]: Test `test_always_fails` failed + --> test.py:2:5 + | + 2 | def test_always_fails(): assert False + | ^^^^^^^^^^^^^^^^^ + | + info: Test failed here + --> test.py:2:1 + | + 2 | def test_always_fails(): assert False + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + | + + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + "); +} + +/// `--max-fail` must reject zero because the underlying type is `NonZeroU32`. +#[test] +fn test_max_fail_zero_is_rejected() { + let context = TestContext::with_file("test.py", "def test_1(): pass"); + + assert_cmd_snapshot!(context.command().args(["--max-fail", "0"]), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: invalid value '0' for '--max-fail ': number would be zero for non-zero type + + For more information, try '--help'. + "); +} + +/// `--num-workers` followed by a non-numeric value should trigger clap's parser. +#[test] +fn test_num_workers_invalid_value() { + let context = TestContext::with_file("test.py", "def test_1(): pass"); + + assert_cmd_snapshot!(context.command().args(["--num-workers", "abc"]), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: invalid value 'abc' for '--num-workers ': invalid digit found in string + + For more information, try '--help'. + "); +} + +/// `--num-workers 1` behaves like `--no-parallel`: one worker handles every test. +#[test] +fn test_num_workers_one_matches_no_parallel() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): pass +def test_2(): pass +", + ); + + assert_cmd_snapshot!(context.command().args(["--num-workers", "1"]), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 2 tests across 1 worker + PASS [TIME] test::test_1 + PASS [TIME] test::test_2 + + ──────────── + Summary [TIME] 2 tests run: 2 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// `--durations` requires a numeric argument. +#[test] +fn test_durations_invalid_value() { + let context = TestContext::with_file("test.py", "def test_1(): pass"); + + assert_cmd_snapshot!(context.command().args(["--durations", "abc"]), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: invalid value 'abc' for '--durations ': invalid digit found in string + + For more information, try '--help'. + "); +} + +/// `--dry-run` with `--num-workers` still only collects — it should not spawn workers. +#[test] +fn test_dry_run_with_num_workers_does_not_spawn() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): pass +def test_2(): pass +", + ); + + assert_cmd_snapshot!(context.command().args(["--dry-run", "--num-workers", "4"]), @r" + success: true + exit_code: 0 + ----- stdout ----- + test::test_1 + test::test_2 + + 2 tests collected + + ----- stderr ----- + "); +} + +/// When `--fail-fast` and `--no-fail-fast` are mixed, clap's `overrides_with` +/// wires them so that whichever flag appears last wins. +#[test] +fn test_no_fail_fast_after_fail_fast_wins() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): assert False +def test_2(): assert False +def test_3(): pass +", + ); + + assert_cmd_snapshot!( + context + .command_no_parallel() + .args(["--fail-fast", "--no-fail-fast", "-q"]), + @" + success: false + exit_code: 1 + ----- stdout ----- + ──────────── + Summary [TIME] 3 tests run: 1 passed, 2 failed, 0 skipped + + ----- stderr ----- + " + ); +} + +#[test] +fn test_fail_fast_after_no_fail_fast_wins() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): assert False +def test_2(): assert False +def test_3(): pass +", + ); + + assert_cmd_snapshot!( + context + .command_no_parallel() + .args(["--no-fail-fast", "--fail-fast", "-q"]), + @" + success: false + exit_code: 1 + ----- stdout ----- + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + " + ); +} + +/// `--max-fail` wins over `--no-fail-fast` regardless of order. +#[test] +fn test_max_fail_beats_no_fail_fast() { + let context = TestContext::with_file( + "test.py", + r" +def test_1(): assert False +def test_2(): assert False +def test_3(): assert False +", + ); + + assert_cmd_snapshot!( + context + .command_no_parallel() + .args(["--no-fail-fast", "--max-fail=2", "-q"]), + @" + success: false + exit_code: 1 + ----- stdout ----- + ──────────── + Summary [TIME] 2 tests run: 0 passed, 2 failed, 0 skipped + + ----- stderr ----- + " + ); +} + +/// `karva test nonexistent.py` should exit with code 2 and an error message +/// that points at the missing path. +#[test] +fn test_nonexistent_path_exits_nonzero() { + let context = TestContext::new(); + + assert_cmd_snapshot!(context.command().arg("missing.py"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Karva failed + Cause: path `/missing.py` could not be found + "); +} + +/// `karva` with no subcommand should print help and exit successfully. +#[test] +fn test_no_subcommand_prints_help() { + let context = TestContext::new(); + + let output = context + .karva_command_in(context.root()) + .output() + .expect("failed to run karva with no subcommand"); + + // No subcommand is a clap error; exit code is 2 and help goes to stderr. + assert_eq!(output.status.code(), Some(2)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("Usage: karva "), + "expected usage line in stderr, got: {stderr}" + ); +} + +/// `karva testx` (typo of `test`) should suggest the closest subcommand. +#[test] +fn test_unknown_subcommand_suggests_correction() { + let context = TestContext::new(); + + let mut command = context.karva_command_in(context.root()); + command.arg("testx"); + + assert_cmd_snapshot!(command, @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: unrecognized subcommand 'testx' + + tip: a similar subcommand exists: 'test' + + Usage: karva + + For more information, try '--help'. + "); +} + +/// `--test-prefix` requires a value. +#[test] +fn test_test_prefix_requires_value() { + let context = TestContext::with_file("test.py", "def test_1(): pass"); + + assert_cmd_snapshot!(context.command().arg("--test-prefix"), @r" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + error: a value is required for '--test-prefix ' but none was supplied + + For more information, try '--help'. + "); +} diff --git a/crates/karva/tests/it/configuration/mod.rs b/crates/karva/tests/it/configuration/mod.rs index 0a9f7b68..7b0d5398 100644 --- a/crates/karva/tests/it/configuration/mod.rs +++ b/crates/karva/tests/it/configuration/mod.rs @@ -1379,6 +1379,130 @@ def test_should_not_run(): pass "); } +/// The `KARVA_CONFIG_FILE` environment variable is equivalent to passing +/// `--config-file` on the command line. +#[test] +fn test_config_file_env_var() { + let context = TestContext::with_files([ + ( + "custom.toml", + r#" +[test] +test-function-prefix = "spec" +"#, + ), + ( + "test.py", + r" +def spec_example(): pass +def test_should_not_run(): pass +", + ), + ]); + + assert_cmd_snapshot!( + context.command().env("KARVA_CONFIG_FILE", "custom.toml"), + @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test::spec_example + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + " + ); +} + +/// An explicit `--config-file` takes precedence over the `KARVA_CONFIG_FILE` +/// environment variable. +#[test] +fn test_cli_config_file_overrides_env() { + let context = TestContext::with_files([ + ( + "env.toml", + r#" +[test] +test-function-prefix = "env" +"#, + ), + ( + "cli.toml", + r#" +[test] +test-function-prefix = "cli" +"#, + ), + ( + "test.py", + r" +def env_should_not_run(): pass +def cli_should_run(): pass +", + ), + ]); + + assert_cmd_snapshot!( + context + .command() + .env("KARVA_CONFIG_FILE", "env.toml") + .args(["--config-file", "cli.toml"]), + @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test::cli_should_run + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + " + ); +} + +/// `karva.toml` discovered from a parent directory should still apply when +/// karva is invoked from a subdirectory. +#[test] +fn test_karva_toml_discovered_from_subdirectory() { + let context = TestContext::with_files([ + ( + "karva.toml", + r#" +[test] +test-function-prefix = "check" +"#, + ), + ( + "tests/test_a.py", + r" +def check_found(): pass +def test_should_not_run(): pass +", + ), + ]); + + let mut cmd = context.karva_command_in(context.root().join("tests")); + cmd.arg("test"); + + assert_cmd_snapshot!(cmd, @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] tests.test_a::check_found + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + #[test] #[cfg(unix)] fn test_config_file_flag_nonexistent_unix() { diff --git a/crates/karva/tests/it/discovery/edge_cases.rs b/crates/karva/tests/it/discovery/edge_cases.rs new file mode 100644 index 00000000..7859d27f --- /dev/null +++ b/crates/karva/tests/it/discovery/edge_cases.rs @@ -0,0 +1,184 @@ +use insta_cmd::assert_cmd_snapshot; + +use crate::common::TestContext; + +/// `__pycache__` directories and compiled `.pyc` files alongside source files +/// should not be picked up as tests. +#[test] +fn test_pyc_files_and_pycache_are_ignored() { + let context = TestContext::with_files([( + "test_real.py", + r" +def test_real(): pass +", + )]); + + let pycache = context.root().join("__pycache__"); + std::fs::create_dir_all(&pycache).expect("failed to create __pycache__"); + std::fs::write(pycache.join("test_real.cpython-313.pyc"), b"bogus") + .expect("failed to write .pyc"); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test_real::test_real + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// A package with `__init__.py` should have its tests discovered under the +/// package path, while standalone sibling files stay at the top level. +#[test] +fn test_package_init_and_standalone_siblings() { + let context = TestContext::with_files([ + ("pkg/__init__.py", ""), + ( + "pkg/test_in_pkg.py", + r" +def test_inside_package(): pass +", + ), + ( + "test_standalone.py", + r" +def test_at_root(): pass +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel().arg("-q"), @" + success: true + exit_code: 0 + ----- stdout ----- + ──────────── + Summary [TIME] 2 tests run: 2 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// A test directory matching a `.gitignore` rule is skipped by default and +/// restored when `--no-ignore` is passed. +#[test] +fn test_gitignore_excludes_directory() { + let context = TestContext::with_files([ + (".gitignore", "ignored/\n"), + ( + "ignored/test_skipped.py", + r" +def test_skipped(): pass +", + ), + ( + "test_kept.py", + r" +def test_kept(): pass +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test_kept::test_kept + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + +#[test] +fn test_no_ignore_includes_gitignored_directory() { + let context = TestContext::with_files([ + (".gitignore", "ignored/\n"), + ( + "ignored/test_skipped.py", + r" +def test_in_ignored(): pass +", + ), + ( + "test_kept.py", + r" +def test_kept(): pass +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel().args(["--no-ignore", "-q"]), @" + success: true + exit_code: 0 + ----- stdout ----- + ──────────── + Summary [TIME] 2 tests run: 2 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// A python file that contains no test functions alongside a file that does +/// should be collected silently. +#[test] +fn test_python_file_without_test_functions_is_ignored() { + let context = TestContext::with_files([ + ( + "test_helpers.py", + r" +x = 1 +def helper(): + return 42 +", + ), + ( + "test_real.py", + r" +def test_one(): pass +", + ), + ]); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test_real::test_one + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} + +/// An empty subdirectory (no Python files at all) is discovered without error. +#[test] +fn test_empty_subdirectory_is_ignored() { + let context = TestContext::with_file("test_a.py", "def test_a(): pass"); + + std::fs::create_dir_all(context.root().join("empty_dir")) + .expect("failed to create empty directory"); + + assert_cmd_snapshot!(context.command_no_parallel(), @" + success: true + exit_code: 0 + ----- stdout ----- + Starting 1 test across 1 worker + PASS [TIME] test_a::test_a + + ──────────── + Summary [TIME] 1 test run: 1 passed, 0 skipped + + ----- stderr ----- + "); +} diff --git a/crates/karva/tests/it/discovery/mod.rs b/crates/karva/tests/it/discovery/mod.rs index 13113268..8350d1d4 100644 --- a/crates/karva/tests/it/discovery/mod.rs +++ b/crates/karva/tests/it/discovery/mod.rs @@ -1,2 +1,3 @@ +mod edge_cases; mod git_boundary; mod nested_layouts; diff --git a/crates/karva/tests/it/last_failed.rs b/crates/karva/tests/it/last_failed.rs index fabc92b3..650a2ad2 100644 --- a/crates/karva/tests/it/last_failed.rs +++ b/crates/karva/tests/it/last_failed.rs @@ -172,6 +172,159 @@ def test_fail_b(): assert False "); } +/// `--dry-run` ignores `--last-failed` and prints every discovered test. +#[test] +fn last_failed_with_dry_run_shows_all_tests() { + let context = TestContext::with_file( + "test_a.py", + " +def test_pass(): pass +def test_fail(): assert False + ", + ); + + context.command_no_parallel().output().unwrap(); + + assert_cmd_snapshot!( + context + .command_no_parallel() + .args(["--last-failed", "--dry-run"]), + @" + success: true + exit_code: 0 + ----- stdout ----- + test_a::test_fail + test_a::test_pass + + 2 tests collected + + ----- stderr ----- + " + ); +} + +/// A filter combined with `--last-failed` intersects: tests that were in the +/// last-failed set but are now filtered out are skipped. +#[test] +fn last_failed_with_filter_intersects() { + let context = TestContext::with_file( + "test_a.py", + " +def test_pass(): pass +def test_fail_a(): assert False +def test_fail_b(): assert False + ", + ); + + context.command_no_parallel().output().unwrap(); + + assert_cmd_snapshot!( + context + .command_no_parallel() + .args(["--last-failed", "-E", "test(~fail_a)"]), + @" + success: false + exit_code: 1 + ----- stdout ----- + Starting 3 tests across 1 worker + FAIL [TIME] test_a::test_fail_a + SKIP [TIME] test_a::test_fail_b + + diagnostics: + + error[test-failure]: Test `test_fail_a` failed + --> test_a.py:3:5 + | + 2 | def test_pass(): pass + 3 | def test_fail_a(): assert False + | ^^^^^^^^^^^ + 4 | def test_fail_b(): assert False + | + info: Test failed here + --> test_a.py:3:1 + | + 2 | def test_pass(): pass + 3 | def test_fail_a(): assert False + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + 4 | def test_fail_b(): assert False + | + + ──────────── + Summary [TIME] 2 tests run: 0 passed, 1 failed, 1 skipped + + ----- stderr ----- + " + ); +} + +/// `--last-failed` + `--max-fail=1` still stops scheduling once a single test +/// in the rerun has failed. +#[test] +fn last_failed_with_max_fail_stops_early() { + let context = TestContext::with_file( + "test_a.py", + " +def test_pass(): pass +def test_fail_a(): assert False +def test_fail_b(): assert False + ", + ); + + context.command_no_parallel().output().unwrap(); + + assert_cmd_snapshot!( + context + .command_no_parallel() + .args(["--last-failed", "--max-fail=1", "-q"]), + @" + success: false + exit_code: 1 + ----- stdout ----- + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + " + ); +} + +/// Adding a brand new test after a run does not cause `--last-failed` to pick +/// it up — only previously-known failures are rerun. +#[test] +fn last_failed_ignores_newly_added_tests() { + let context = TestContext::with_file( + "test_a.py", + " +def test_pass(): pass +def test_fail(): assert False + ", + ); + + context.command_no_parallel().output().unwrap(); + + context.write_file( + "test_a.py", + " +def test_pass(): pass +def test_fail(): assert False +def test_new_fail(): assert False + ", + ); + + assert_cmd_snapshot!( + context.command_no_parallel().args(["--last-failed", "-q"]), + @" + success: false + exit_code: 1 + ----- stdout ----- + ──────────── + Summary [TIME] 1 test run: 0 passed, 1 failed, 0 skipped + + ----- stderr ----- + " + ); +} + #[test] fn last_failed_fix_then_rerun() { let context = TestContext::with_file(