diff --git a/tests/integration/crash_on_startup/BUILD b/tests/integration/crash_on_startup/BUILD index 1a74dcca74..c416fc6b2b 100644 --- a/tests/integration/crash_on_startup/BUILD +++ b/tests/integration/crash_on_startup/BUILD @@ -24,29 +24,13 @@ cc_binary( ], ) -cc_binary( - name = "process_crashing_on_startup_twice", - srcs = ["process_crashing_on_startup_twice.cpp"], - deps = [ - "//score/launch_manager:lifecycle_cc", - "//tests/utils/test_helper", - "@googletest//:gtest_main", - ], -) - -cc_binary( - name = "process_crashing_on_startup_always", - srcs = ["process_crashing_on_startup_always.cpp"], -) - integration_test( name = "crash_on_startup", srcs = ["crash_on_startup.py"], binaries = [ ":control_client_mock", - ":process_crashing_on_startup_always", - ":process_crashing_on_startup_twice", "//score/launch_manager", + "//tests/utils/test_helper:process_crashing_on_startup_n_times", "//tests/utils/test_helper:verification_process", ], config = ":crash_on_startup.json", diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index abc39de307..f8a5e3089f 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -21,34 +21,42 @@ TEST(CrashOnStartup, ControlClientMock) { score::mw::lifecycle::ControlClient client; - ASSERT_TRUE(check_clean({crashed_once_file, crashed_twice_file, test_end_location, fallback_file})); + ASSERT_TRUE( + check_clean({crashCountPath(1), crashCountPath(2), crashCountPath(3), test_end_location, fallback_file})); TEST_STEP("Report running") { score::mw::lifecycle::report_running(); } - // Given a process that crashes on startup twice - TEST_STEP("Launch process crashing on startup twice") + // Given a process that crashes on startup n times, but is configured to retry n times - so it eventually + // succeeds. The behaviour is identical for the different crash counts, so it is parameterized over the + // corresponding run targets. Each run target's process persists its crash count in its own file, so the + // run targets do not interfere with each other. + for (const std::string_view run_target : + {"run_target_crash_on_startup_two_times", "run_target_crash_on_startup_three_times"}) { - score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_twice").Get(stop_token); - // Then, the LM should restart it and eventually succeed - EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_twice failed: " - << result.error().Message(); - } + TEST_STEP(std::string{"Launch "} + std::string{run_target}) + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget(run_target).Get(stop_token); + // Then, the LM should restart it and eventually succeed + EXPECT_TRUE(result.has_value()) << "Activating " << run_target << " failed: " << result.error().Message(); + } - TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully") - { - EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; + TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully") + { + EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; + } } - // Given a process that crashes on startup more times than the configured restart attempts - TEST_STEP("Attempt to launch process crashing on startup always") + // Given a process that crashes on startup but is not allowed to retry (number_of_attempts=0) + TEST_STEP("Attempt to launch process crashing on startup without retries") { score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_always").Get(stop_token); - EXPECT_FALSE(result.has_value()) << "Expected run_target_crash_on_startup_always activation to fail"; + auto result = client.ActivateRunTarget("run_target_crash_on_startup_once_but_no_retries").Get(stop_token); + EXPECT_FALSE(result.has_value()) + << "Expected run_target_crash_on_startup_once_but_no_retries activation to fail"; } // Limitation: we cannot wait for the transition to fallback to complete sleep(1); diff --git a/tests/integration/crash_on_startup/crash_on_startup.json b/tests/integration/crash_on_startup/crash_on_startup.json index 4ee6fc2bcb..124aed307a 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.json +++ b/tests/integration/crash_on_startup/crash_on_startup.json @@ -3,8 +3,6 @@ "defaults": { "deployment_config": { "bin_dir": "/tmp/tests/crash_on_startup", - "ready_timeout": 1.0, - "shutdown_timeout": 1.0, "ready_recovery_action": { "restart": { "number_of_attempts": 0 @@ -24,21 +22,6 @@ "scheduling_policy": "SCHED_OTHER", "scheduling_priority": 0 } - }, - "component_properties": { - "application_profile": { - "application_type": "Reporting", - "is_self_terminating": false, - "alive_supervision": { - "reporting_cycle": 0.1, - "min_indications": 1, - "max_indications": 3, - "failed_cycles_tolerance": 1 - } - }, - "ready_condition": { - "process_state": "Running" - } } }, "components": { @@ -62,40 +45,51 @@ }, "process_crashing_on_startup_twice": { "component_properties": { - "binary_name": "process_crashing_on_startup_twice", + "binary_name": "process_crashing_on_startup_n_times", "application_profile": { "application_type": "Reporting" - } + }, + "process_arguments": [ + "2" + ] }, "deployment_config": { "ready_recovery_action": { "restart": { "number_of_attempts": 2 } - }, - "environmental_variables": { - "PROCESSIDENTIFIER": "process_crashing_on_startup_twice" } } }, - "process_crashing_on_startup_always": { + "process_crashing_on_startup_three_times": { "component_properties": { - "binary_name": "process_crashing_on_startup_always", + "binary_name": "process_crashing_on_startup_n_times", "application_profile": { "application_type": "Reporting" - } + }, + "process_arguments": [ + "3" + ] }, "deployment_config": { "ready_recovery_action": { "restart": { - "number_of_attempts": 2 + "number_of_attempts": 3 } - }, - "environmental_variables": { - "PROCESSIDENTIFIER": "process_crashing_on_startup_always" } } }, + "process_crashing_on_startup_no_retries": { + "component_properties": { + "binary_name": "process_crashing_on_startup_n_times", + "application_profile": { + "application_type": "Reporting" + }, + "process_arguments": [ + "1" + ] + } + }, "verification_component": { "component_properties": { "binary_name": "verification_process", @@ -118,16 +112,22 @@ "control_client_mock" ] }, - "run_target_crash_on_startup_twice": { + "run_target_crash_on_startup_two_times": { "depends_on": [ "control_client_mock", "process_crashing_on_startup_twice" ] }, - "run_target_crash_on_startup_always": { + "run_target_crash_on_startup_three_times": { + "depends_on": [ + "control_client_mock", + "process_crashing_on_startup_three_times" + ] + }, + "run_target_crash_on_startup_once_but_no_retries": { "depends_on": [ "control_client_mock", - "process_crashing_on_startup_always" + "process_crashing_on_startup_no_retries" ], "recovery_action": { "switch_run_target": { diff --git a/tests/integration/crash_on_startup/crash_on_startup.py b/tests/integration/crash_on_startup/crash_on_startup.py index 29eaaf0706..2a074bed76 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.py +++ b/tests/integration/crash_on_startup/crash_on_startup.py @@ -12,28 +12,36 @@ # ******************************************************************************* from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed from tests.utils.testing_utils.setup_test import setup_test -from tests.utils.testing_utils.test_results import assert_test_results +from tests.utils.testing_utils.test_results import ( + assert_test_results, + get_testcase_property, +) from attribute_plugin import add_test_properties @add_test_properties( fully_verifies=[ "comp_req__launch_man__failure_detect", + "comp_req__launch_man__retries_configurable", "feat_req__lifecycle__recov_run_target_switch", ], partially_verifies=["feat_req__lifecycle__recovery_action_support"], test_type="requirements-based", derivation_technique="requirements-analysis", ) -def test_crash_on_startup(target, setup_test, assert_test_results, remote_test_dir): +def test_crash_on_startup( + target, setup_test, assert_test_results, remote_test_dir, test_output_dir +): """ Objective: Verifies that the launch manager correctly handles processes that crash before reporting running. Case 1: Process crashes before Running state but eventually starts up successfully before the configured number of restart attempts is exceeded. - Expected Behaviour: Process startup successful, RunTarget activation successful + This is verified with two different components: One with the process crashing twice and the other three times before successfully starting up. + The number of restart attempts is configured to be 2 and 3 respectively for these two components. + Expected Behaviour: Process startup successful, run target activation successful - Case 2: Process keeps crashing, exceeding the number of restart attempts. - Expected Behaviour: Process startup fails, LaunchManager executes recovery action. + Case 2: Component has no restart attempts configured, but crashes once. + Expected Behaviour: Process startup fails and therefore run target activation fails. Launch manager executes recovery action which switches to fallback run target. """ config_path = str(remote_test_dir / "etc/crash_on_startup.bin") @@ -47,6 +55,19 @@ def test_crash_on_startup(target, setup_test, assert_test_results, remote_test_d timeout_s=10.0, ) - assert_test_results( - {"control_client_mock.xml", "process_crashing_on_startup_twice.xml"} - ) + # Each crashing process writes its own report file named after the number of times it crashes, so the + # reports of the different run targets no longer overwrite each other. The process crashing once is not + # allowed to retry, but still writes its report before crashing. + crash_report_names = { + n: f"process_crashing_on_startup_n_times_n_equals_{n}.xml" for n in (1, 2, 3) + } + assert_test_results({"control_client_mock.xml", *crash_report_names.values()}) + + # The number of crashes is recorded in each report and must match the configured crash count. + for n, report_name in crash_report_names.items(): + crash_count = get_testcase_property( + test_output_dir / report_name, "crash_count" + ) + assert crash_count == str(n), ( + f"Expected {n} crashes in {report_name}, got {crash_count}" + ) diff --git a/tests/integration/crash_on_startup/process_crashing_on_startup_always.cpp b/tests/integration/crash_on_startup/process_crashing_on_startup_always.cpp deleted file mode 100644 index b0df435556..0000000000 --- a/tests/integration/crash_on_startup/process_crashing_on_startup_always.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - ********************************************************************************/ -#include - -int main() -{ - std::cout << "Process crashing on startup (always)..." << std::endl; - std::abort(); -} diff --git a/tests/integration/crash_on_startup/process_crashing_on_startup_twice.cpp b/tests/integration/crash_on_startup/process_crashing_on_startup_twice.cpp deleted file mode 100644 index 473b270ee8..0000000000 --- a/tests/integration/crash_on_startup/process_crashing_on_startup_twice.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - ********************************************************************************/ -#include -#include -#include - -#include "tests/utils/test_helper/test_helper.hpp" -#include - -TEST(CrashOnStartup, ProcessCrashingOnStartupTwice) -{ - TEST_STEP("Report running") - { - score::mw::lifecycle::report_running(); - } -} - -void deploy_and_crash_if_not_present(const std::string_view name) -{ - if (!std::filesystem::exists(name)) - { - std::cout << "Process crashing on startup..." << std::endl; - if (!touch_file(name)) - { - std::cout << "Failed to deploy marker file!" << std::endl; - } - std::abort(); - } -} - -int main() -{ - deploy_and_crash_if_not_present(crashed_once_file); - deploy_and_crash_if_not_present(crashed_twice_file); - - std::cout << "Process starting successfully..." << std::endl; - return TestRunner(__FILE__).RunTests(); -} diff --git a/tests/utils/test_helper/BUILD b/tests/utils/test_helper/BUILD index d5ef0cde5b..c044cce2cf 100644 --- a/tests/utils/test_helper/BUILD +++ b/tests/utils/test_helper/BUILD @@ -63,3 +63,14 @@ cc_binary( ":test_helper", ], ) + +cc_binary( + name = "process_crashing_on_startup_n_times", + srcs = ["process_crashing_on_startup_n_times.cpp"], + visibility = ["//tests:__subpackages__"], + deps = [ + ":test_helper", + "//score/launch_manager:lifecycle_cc", + "@googletest//:gtest_main", + ], +) diff --git a/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp new file mode 100644 index 0000000000..c88dcafc04 --- /dev/null +++ b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp @@ -0,0 +1,116 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include +#include +#include +#include +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include + +namespace +{ +/// @brief Default number of times the process crashes before starting up successfully. +constexpr int kDefaultCrashesUntilSuccess = 3; + +/// @brief Reads how many times the process has crashed so far from the crash count file. +int read_crash_count(const std::string_view file_path) +{ + std::ifstream file{std::string{file_path}}; + int count = 0; + if (file >> count) + { + return count; + } + return 0; +} + +/// @brief Persists how many times the process has crashed so far to the crash count file. +void write_crash_count(const std::string_view file_path, const int count) +{ + std::ofstream file{std::string{file_path}}; + file << count; +} + +/// @brief Whether this invocation reaches the running state. False when simulating a crash on startup, in +/// which case the process must not report running. +bool g_startup_successful = false; + +/// @brief Path to this invocation's crash count file, set in main() from the configured crash count. Every +/// configured crash count gets its own file, so that concurrently configured processes do not share state. +std::string g_crash_count_path; +} // namespace + +TEST(CrashOnStartup, ProcessCrashingOnStartupNTimes) +{ + TEST_STEP("Record number of crashes so far") + { + // The crash count file holds the number of crashes that occurred, including the crash simulated by + // this invocation (if any). + RecordProperty("crash_count", read_crash_count(g_crash_count_path)); + } + + if (!g_startup_successful) + { + // This invocation simulates a crash before reaching the running state, so it must not report + // running. The process aborts in main() once this report has been written. + return; + } + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } +} + +int main(int argc, char** argv) +{ + if (argc > 1 && (std::string_view{argv[1]} == "-h" || std::string_view{argv[1]} == "--help")) + { + std::cout << "Usage: " << argv[0] << " [crashes_until_success]\n" + << "Crashes on startup the given number of times (default " << kDefaultCrashesUntilSuccess + << ") before starting up successfully.\n" + << "Each configuration persists its crash count and writes its own report in files suffixed " + << "with '_'." << std::endl; + return 0; + } + + // The number of crashes before a successful startup is taken from the command line, defaulting to + // kDefaultCrashesUntilSuccess. + const int crashes_until_success = (argc > 1) ? std::atoi(argv[1]) : kDefaultCrashesUntilSuccess; + + // Each configuration writes its own report file, e.g. process_crashing_on_startup_n_times_n_equals_2.xml, + // so that the report of a process crashing n times is not overwritten by one crashing a different number of times. + const std::string report_path = + std::filesystem::path{__FILE__}.stem().string() + "_n_equals_" + std::to_string(crashes_until_success); + g_crash_count_path = crashCountPath(crashes_until_success); + + const int crash_count = read_crash_count(g_crash_count_path); + if (crash_count < crashes_until_success) + { + std::cout << "Process crashing on startup (" << (crash_count + 1) << "/" << crashes_until_success << ")..." + << std::endl; + write_crash_count(g_crash_count_path, crash_count + 1); + // Write the report before crashing, so that even a process that is never allowed to start up + // successfully (e.g. no retries) still produces its report. + TestRunner(report_path, TerminationBehavior::kContinue).RunTests(); + std::abort(); + } + + std::cout << "Process starting successfully..." << std::endl; + g_startup_successful = true; + return TestRunner(report_path).RunTests(); +} diff --git a/tests/utils/test_helper/test_helper.hpp b/tests/utils/test_helper/test_helper.hpp index ff7c397478..569ec7f1b6 100644 --- a/tests/utils/test_helper/test_helper.hpp +++ b/tests/utils/test_helper/test_helper.hpp @@ -46,13 +46,15 @@ inline testing::AssertionResult touch_file(const std::string_view file_path) /// @brief Location to store a file signalling that the fallback state has been reached. constexpr std::string_view fallback_file = "fallback_reached"; -/// @brief Location to store a file signalling that a process has been killed on first try - used to test ready recovery -/// action -constexpr std::string_view crashed_once_file = "crashed_once"; - -/// @brief Location to store a file signalling that a process has been killed on second try - used to test ready +/// @brief Prefix of the file storing the number of times a process has crashed so far - used to test ready /// recovery action -constexpr std::string_view crashed_twice_file = "crashed_twice"; +constexpr std::string_view crash_count_file = "crash_count"; + +/// @return File path to store the crash count for a process configured to crash `crashes_until_success` times +inline std::string crashCountPath(const int crashes_until_success) +{ + return std::string{crash_count_file} + "_" + std::to_string(crashes_until_success); +} /// @brief Where to store the test_end signal file. This must be kept consistent with where the test framework /// searches for files. diff --git a/tests/utils/testing_utils/test_results.py b/tests/utils/testing_utils/test_results.py index 3eff9c29b6..8562d77945 100644 --- a/tests/utils/testing_utils/test_results.py +++ b/tests/utils/testing_utils/test_results.py @@ -68,6 +68,16 @@ def get_failing_files(path: Path): return all_files, failing_files +def get_testcase_property(path: Path, name: str) -> str: + """Returns the value of the `name` property recorded on the first testcase of the + given xml result file. gtest emits properties recorded via RecordProperty as + `` elements nested in the ``.""" + root = ElementTree.parse(str(path)).getroot() + prop = root.find(f".//testcase/properties/property[@name='{name}']") + assert prop is not None, f"Property '{name}' not found in {path.name}" + return prop.get("value") + + @pytest.fixture def assert_test_results(target, remote_test_dir, test_output_dir): """Returns a callable that downloads XML results and asserts the expected