From f537520a159a343eada8bce4a475d456dd315569 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Thu, 16 Jul 2026 14:57:07 +0200 Subject: [PATCH 01/12] Update test binary to crash n times, depending on param --- tests/integration/crash_on_startup/BUILD | 6 +- .../crash_on_startup/control_client_mock.cpp | 2 +- .../crash_on_startup/crash_on_startup.json | 9 ++- .../crash_on_startup/crash_on_startup.py | 3 +- .../process_crashing_on_startup_n_times.cpp | 69 +++++++++++++++++++ .../process_crashing_on_startup_twice.cpp | 48 ------------- tests/utils/test_helper/test_helper.hpp | 9 +-- 7 files changed, 83 insertions(+), 63 deletions(-) create mode 100644 tests/integration/crash_on_startup/process_crashing_on_startup_n_times.cpp delete mode 100644 tests/integration/crash_on_startup/process_crashing_on_startup_twice.cpp diff --git a/tests/integration/crash_on_startup/BUILD b/tests/integration/crash_on_startup/BUILD index 1a74dcca74..6695f191d5 100644 --- a/tests/integration/crash_on_startup/BUILD +++ b/tests/integration/crash_on_startup/BUILD @@ -25,8 +25,8 @@ cc_binary( ) cc_binary( - name = "process_crashing_on_startup_twice", - srcs = ["process_crashing_on_startup_twice.cpp"], + name = "process_crashing_on_startup_n_times", + srcs = ["process_crashing_on_startup_n_times.cpp"], deps = [ "//score/launch_manager:lifecycle_cc", "//tests/utils/test_helper", @@ -45,7 +45,7 @@ integration_test( binaries = [ ":control_client_mock", ":process_crashing_on_startup_always", - ":process_crashing_on_startup_twice", + ":process_crashing_on_startup_n_times", "//score/launch_manager", "//tests/utils/test_helper:verification_process", ], diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index abc39de307..6d8e517d7f 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -21,7 +21,7 @@ TEST(CrashOnStartup, ControlClientMock) { score::mw::lifecycle::ControlClient client; - ASSERT_TRUE(check_clean({crashed_once_file, crashed_twice_file, test_end_location, fallback_file})); + ASSERT_TRUE(check_clean({crash_count_file, test_end_location, fallback_file})); TEST_STEP("Report running") { diff --git a/tests/integration/crash_on_startup/crash_on_startup.json b/tests/integration/crash_on_startup/crash_on_startup.json index 4ee6fc2bcb..6648792f7d 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.json +++ b/tests/integration/crash_on_startup/crash_on_startup.json @@ -62,10 +62,13 @@ }, "process_crashing_on_startup_twice": { "component_properties": { - "binary_name": "process_crashing_on_startup_twice", + "binary_name": "process_crashing_on_startup_n_times", "application_profile": { "application_type": "Reporting" - } + }, + "process_arguments": [ + "2" + ] }, "deployment_config": { "ready_recovery_action": { @@ -74,7 +77,7 @@ } }, "environmental_variables": { - "PROCESSIDENTIFIER": "process_crashing_on_startup_twice" + "PROCESSIDENTIFIER": "process_crashing_on_startup_n_times" } } }, diff --git a/tests/integration/crash_on_startup/crash_on_startup.py b/tests/integration/crash_on_startup/crash_on_startup.py index 29eaaf0706..a524fb2143 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.py +++ b/tests/integration/crash_on_startup/crash_on_startup.py @@ -19,6 +19,7 @@ @add_test_properties( fully_verifies=[ "comp_req__launch_man__failure_detect", + "feat_req__lifecycle__retries_configurable", #TODO: Add another process with another amount of retries configured to fully verify requirement "feat_req__lifecycle__recov_run_target_switch", ], partially_verifies=["feat_req__lifecycle__recovery_action_support"], @@ -48,5 +49,5 @@ def test_crash_on_startup(target, setup_test, assert_test_results, remote_test_d ) assert_test_results( - {"control_client_mock.xml", "process_crashing_on_startup_twice.xml"} + {"control_client_mock.xml", "process_crashing_on_startup_n_times.xml"} ) diff --git a/tests/integration/crash_on_startup/process_crashing_on_startup_n_times.cpp b/tests/integration/crash_on_startup/process_crashing_on_startup_n_times.cpp new file mode 100644 index 0000000000..230bb37aea --- /dev/null +++ b/tests/integration/crash_on_startup/process_crashing_on_startup_n_times.cpp @@ -0,0 +1,69 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include +#include +#include +#include +#include +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include + +/// @brief Default number of times the process crashes before starting up successfully. +constexpr int kDefaultCrashesUntilSuccess = 3; + +TEST(CrashOnStartup, ProcessCrashingOnStartupTwice) +{ + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } +} + +/// @brief Reads how many times the process has crashed so far from the crash count file. +int read_crash_count(const std::string_view file_path) +{ + std::ifstream file{std::string{file_path}}; + int count = 0; + if (file >> count) + { + return count; + } + return 0; +} + +/// @brief Persists how many times the process has crashed so far to the crash count file. +void write_crash_count(const std::string_view file_path, const int count) +{ + std::ofstream file{std::string{file_path}}; + file << count; +} + +int main(int argc, char** argv) +{ + // The number of crashes before a successful startup is taken from the command line, defaulting to two. + const int crashes_until_success = (argc > 1) ? std::atoi(argv[1]) : kDefaultCrashesUntilSuccess; + + const int crash_count = read_crash_count(crash_count_file); + if (crash_count < crashes_until_success) + { + std::cout << "Process crashing on startup (" << (crash_count + 1) << "/" << crashes_until_success << ")..." + << std::endl; + write_crash_count(crash_count_file, crash_count + 1); + std::abort(); + } + + std::cout << "Process starting successfully..." << std::endl; + return TestRunner(__FILE__).RunTests(); +} diff --git a/tests/integration/crash_on_startup/process_crashing_on_startup_twice.cpp b/tests/integration/crash_on_startup/process_crashing_on_startup_twice.cpp deleted file mode 100644 index 473b270ee8..0000000000 --- a/tests/integration/crash_on_startup/process_crashing_on_startup_twice.cpp +++ /dev/null @@ -1,48 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - ********************************************************************************/ -#include -#include -#include - -#include "tests/utils/test_helper/test_helper.hpp" -#include - -TEST(CrashOnStartup, ProcessCrashingOnStartupTwice) -{ - TEST_STEP("Report running") - { - score::mw::lifecycle::report_running(); - } -} - -void deploy_and_crash_if_not_present(const std::string_view name) -{ - if (!std::filesystem::exists(name)) - { - std::cout << "Process crashing on startup..." << std::endl; - if (!touch_file(name)) - { - std::cout << "Failed to deploy marker file!" << std::endl; - } - std::abort(); - } -} - -int main() -{ - deploy_and_crash_if_not_present(crashed_once_file); - deploy_and_crash_if_not_present(crashed_twice_file); - - std::cout << "Process starting successfully..." << std::endl; - return TestRunner(__FILE__).RunTests(); -} diff --git a/tests/utils/test_helper/test_helper.hpp b/tests/utils/test_helper/test_helper.hpp index ff7c397478..eb57ccc75c 100644 --- a/tests/utils/test_helper/test_helper.hpp +++ b/tests/utils/test_helper/test_helper.hpp @@ -46,13 +46,8 @@ inline testing::AssertionResult touch_file(const std::string_view file_path) /// @brief Location to store a file signalling that the fallback state has been reached. constexpr std::string_view fallback_file = "fallback_reached"; -/// @brief Location to store a file signalling that a process has been killed on first try - used to test ready recovery -/// action -constexpr std::string_view crashed_once_file = "crashed_once"; - -/// @brief Location to store a file signalling that a process has been killed on second try - used to test ready -/// recovery action -constexpr std::string_view crashed_twice_file = "crashed_twice"; +/// @brief Location to store the number of times a process has crashed so far - used to test ready recovery action +constexpr std::string_view crash_count_file = "crash_count"; /// @brief Where to store the test_end signal file. This must be kept consistent with where the test framework /// searches for files. From 7c0f399e54dd81c32dc4ab6570e470421d3009d1 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Thu, 16 Jul 2026 15:30:56 +0200 Subject: [PATCH 02/12] Add another rt and component which crashes 5 times --- .../crash_on_startup/control_client_mock.cpp | 15 +++++++++++ .../crash_on_startup/crash_on_startup.json | 27 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index 6d8e517d7f..72b1740327 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -43,6 +43,21 @@ TEST(CrashOnStartup, ControlClientMock) EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; } + // Given a process that crashes on startup five times, but is configured to retry five times - so still succeeds + TEST_STEP("Launch process crashing on startup five times") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_crash_on_startup_five_times").Get(stop_token); + // Then, the LM should restart it and eventually succeed + EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_five_times failed: " + << result.error().Message(); + } + + TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully") + { + EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; + } + // Given a process that crashes on startup more times than the configured restart attempts TEST_STEP("Attempt to launch process crashing on startup always") { diff --git a/tests/integration/crash_on_startup/crash_on_startup.json b/tests/integration/crash_on_startup/crash_on_startup.json index 6648792f7d..97682a1072 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.json +++ b/tests/integration/crash_on_startup/crash_on_startup.json @@ -81,6 +81,27 @@ } } }, + "process_crashing_on_startup_five_times": { + "component_properties": { + "binary_name": "process_crashing_on_startup_n_times", + "application_profile": { + "application_type": "Reporting" + }, + "process_arguments": [ + "5" + ] + }, + "deployment_config": { + "ready_recovery_action": { + "restart": { + "number_of_attempts": 5 + } + }, + "environmental_variables": { + "PROCESSIDENTIFIER": "process_crashing_on_startup_n_times" + } + } + }, "process_crashing_on_startup_always": { "component_properties": { "binary_name": "process_crashing_on_startup_always", @@ -127,6 +148,12 @@ "process_crashing_on_startup_twice" ] }, + "run_target_crash_on_startup_five_times": { + "depends_on": [ + "control_client_mock", + "process_crashing_on_startup_five_times" + ] + }, "run_target_crash_on_startup_always": { "depends_on": [ "control_client_mock", From 0104f739c07446505fc51114b65ec911a9a4f6b1 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Thu, 16 Jul 2026 19:00:26 +0200 Subject: [PATCH 03/12] Finish crash_on_startup test --- tests/integration/crash_on_startup/control_client_mock.cpp | 2 ++ tests/integration/crash_on_startup/crash_on_startup.json | 2 +- tests/integration/crash_on_startup/crash_on_startup.py | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index 72b1740327..c71ed31bce 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -43,6 +43,8 @@ TEST(CrashOnStartup, ControlClientMock) EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; } + EXPECT_TRUE(std::filesystem::remove(crash_count_file)) << "Count file must be removed successfully, before reused in the next run target"; + // Given a process that crashes on startup five times, but is configured to retry five times - so still succeeds TEST_STEP("Launch process crashing on startup five times") { diff --git a/tests/integration/crash_on_startup/crash_on_startup.json b/tests/integration/crash_on_startup/crash_on_startup.json index 97682a1072..a8b3484876 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.json +++ b/tests/integration/crash_on_startup/crash_on_startup.json @@ -88,7 +88,7 @@ "application_type": "Reporting" }, "process_arguments": [ - "5" + "5" ] }, "deployment_config": { diff --git a/tests/integration/crash_on_startup/crash_on_startup.py b/tests/integration/crash_on_startup/crash_on_startup.py index a524fb2143..724b35b423 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.py +++ b/tests/integration/crash_on_startup/crash_on_startup.py @@ -19,7 +19,7 @@ @add_test_properties( fully_verifies=[ "comp_req__launch_man__failure_detect", - "feat_req__lifecycle__retries_configurable", #TODO: Add another process with another amount of retries configured to fully verify requirement + "feat_req__lifecycle__retries_configurable", "feat_req__lifecycle__recov_run_target_switch", ], partially_verifies=["feat_req__lifecycle__recovery_action_support"], From 4af6fa1d7033cbd86e60066cb6b921d18542bbe8 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 11:42:59 +0200 Subject: [PATCH 04/12] Requirement is no longer on feature, but on cmp level --- tests/integration/crash_on_startup/crash_on_startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/crash_on_startup/crash_on_startup.py b/tests/integration/crash_on_startup/crash_on_startup.py index 724b35b423..17f4346428 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.py +++ b/tests/integration/crash_on_startup/crash_on_startup.py @@ -19,7 +19,7 @@ @add_test_properties( fully_verifies=[ "comp_req__launch_man__failure_detect", - "feat_req__lifecycle__retries_configurable", + "comp_req__launch_man__retries_configurable", "feat_req__lifecycle__recov_run_target_switch", ], partially_verifies=["feat_req__lifecycle__recovery_action_support"], From edee80cd41b0ab1cda6a63e504b2aee165568613 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 15:29:19 +0200 Subject: [PATCH 05/12] Code review rework, strip down config - Strip down config - Get rid of process_crashing_on_startup_always.cpp - Decrease number of crashes and restarts, but still verify that number_of_attempts is configurable - Move process_crashing_on_startup_n_times.cpp to tests/utils/test_helper folder --- tests/integration/crash_on_startup/BUILD | 18 +----- .../crash_on_startup/control_client_mock.cpp | 31 ++++++---- .../crash_on_startup/crash_on_startup.json | 62 +++++-------------- .../process_crashing_on_startup_always.cpp | 19 ------ tests/utils/test_helper/BUILD | 11 ++++ .../process_crashing_on_startup_n_times.cpp | 15 ++++- 6 files changed, 58 insertions(+), 98 deletions(-) delete mode 100644 tests/integration/crash_on_startup/process_crashing_on_startup_always.cpp rename tests/{integration/crash_on_startup => utils/test_helper}/process_crashing_on_startup_n_times.cpp (78%) diff --git a/tests/integration/crash_on_startup/BUILD b/tests/integration/crash_on_startup/BUILD index 6695f191d5..c416fc6b2b 100644 --- a/tests/integration/crash_on_startup/BUILD +++ b/tests/integration/crash_on_startup/BUILD @@ -24,29 +24,13 @@ cc_binary( ], ) -cc_binary( - name = "process_crashing_on_startup_n_times", - srcs = ["process_crashing_on_startup_n_times.cpp"], - deps = [ - "//score/launch_manager:lifecycle_cc", - "//tests/utils/test_helper", - "@googletest//:gtest_main", - ], -) - -cc_binary( - name = "process_crashing_on_startup_always", - srcs = ["process_crashing_on_startup_always.cpp"], -) - integration_test( name = "crash_on_startup", srcs = ["crash_on_startup.py"], binaries = [ ":control_client_mock", - ":process_crashing_on_startup_always", - ":process_crashing_on_startup_n_times", "//score/launch_manager", + "//tests/utils/test_helper:process_crashing_on_startup_n_times", "//tests/utils/test_helper:verification_process", ], config = ":crash_on_startup.json", diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index c71ed31bce..82bfed318d 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -28,13 +28,13 @@ TEST(CrashOnStartup, ControlClientMock) score::mw::lifecycle::report_running(); } - // Given a process that crashes on startup twice - TEST_STEP("Launch process crashing on startup twice") + // Given a process that crashes on startup once, but is configured to retry twice - so it succeeds + TEST_STEP("Launch process crashing on startup once") { score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_twice").Get(stop_token); + auto result = client.ActivateRunTarget("run_target_crash_on_startup_once").Get(stop_token); // Then, the LM should restart it and eventually succeed - EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_twice failed: " + EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_once failed: " << result.error().Message(); } @@ -43,15 +43,17 @@ TEST(CrashOnStartup, ControlClientMock) EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; } - EXPECT_TRUE(std::filesystem::remove(crash_count_file)) << "Count file must be removed successfully, before reused in the next run target"; + EXPECT_TRUE(std::filesystem::remove(crash_count_file)) + << "Count file must be removed successfully, before reused in the next run target"; - // Given a process that crashes on startup five times, but is configured to retry five times - so still succeeds - TEST_STEP("Launch process crashing on startup five times") + // Given a process that crashes on startup three times, but is configured to retry three times - so it still + // succeeds + TEST_STEP("Launch process crashing on startup three times") { score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_five_times").Get(stop_token); + auto result = client.ActivateRunTarget("run_target_crash_on_startup_three_times").Get(stop_token); // Then, the LM should restart it and eventually succeed - EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_five_times failed: " + EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_three_times failed: " << result.error().Message(); } @@ -60,12 +62,15 @@ TEST(CrashOnStartup, ControlClientMock) EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; } - // Given a process that crashes on startup more times than the configured restart attempts - TEST_STEP("Attempt to launch process crashing on startup always") + EXPECT_TRUE(std::filesystem::remove(crash_count_file)) + << "Count file must be removed successfully, before reused in the next run target"; + + // Given a process that crashes on startup but is not allowed to retry (number_of_attempts=0) + TEST_STEP("Attempt to launch process crashing on startup without retries") { score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_always").Get(stop_token); - EXPECT_FALSE(result.has_value()) << "Expected run_target_crash_on_startup_always activation to fail"; + auto result = client.ActivateRunTarget("run_target_crash_on_startup_no_retries").Get(stop_token); + EXPECT_FALSE(result.has_value()) << "Expected run_target_crash_on_startup_no_retries activation to fail"; } // Limitation: we cannot wait for the transition to fallback to complete sleep(1); diff --git a/tests/integration/crash_on_startup/crash_on_startup.json b/tests/integration/crash_on_startup/crash_on_startup.json index a8b3484876..04b35856ae 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.json +++ b/tests/integration/crash_on_startup/crash_on_startup.json @@ -3,8 +3,6 @@ "defaults": { "deployment_config": { "bin_dir": "/tmp/tests/crash_on_startup", - "ready_timeout": 1.0, - "shutdown_timeout": 1.0, "ready_recovery_action": { "restart": { "number_of_attempts": 0 @@ -24,21 +22,6 @@ "scheduling_policy": "SCHED_OTHER", "scheduling_priority": 0 } - }, - "component_properties": { - "application_profile": { - "application_type": "Reporting", - "is_self_terminating": false, - "alive_supervision": { - "reporting_cycle": 0.1, - "min_indications": 1, - "max_indications": 3, - "failed_cycles_tolerance": 1 - } - }, - "ready_condition": { - "process_state": "Running" - } } }, "components": { @@ -60,14 +43,14 @@ } } }, - "process_crashing_on_startup_twice": { + "process_crashing_on_startup_once": { "component_properties": { "binary_name": "process_crashing_on_startup_n_times", "application_profile": { "application_type": "Reporting" }, "process_arguments": [ - "2" + "1" ] }, "deployment_config": { @@ -75,49 +58,36 @@ "restart": { "number_of_attempts": 2 } - }, - "environmental_variables": { - "PROCESSIDENTIFIER": "process_crashing_on_startup_n_times" } } }, - "process_crashing_on_startup_five_times": { + "process_crashing_on_startup_three_times": { "component_properties": { "binary_name": "process_crashing_on_startup_n_times", "application_profile": { "application_type": "Reporting" }, "process_arguments": [ - "5" + "3" ] }, "deployment_config": { "ready_recovery_action": { "restart": { - "number_of_attempts": 5 + "number_of_attempts": 3 } - }, - "environmental_variables": { - "PROCESSIDENTIFIER": "process_crashing_on_startup_n_times" } } }, - "process_crashing_on_startup_always": { + "process_crashing_on_startup_no_retries": { "component_properties": { - "binary_name": "process_crashing_on_startup_always", + "binary_name": "process_crashing_on_startup_n_times", "application_profile": { "application_type": "Reporting" - } - }, - "deployment_config": { - "ready_recovery_action": { - "restart": { - "number_of_attempts": 2 - } }, - "environmental_variables": { - "PROCESSIDENTIFIER": "process_crashing_on_startup_always" - } + "process_arguments": [ + "1" + ] } }, "verification_component": { @@ -142,22 +112,22 @@ "control_client_mock" ] }, - "run_target_crash_on_startup_twice": { + "run_target_crash_on_startup_once": { "depends_on": [ "control_client_mock", - "process_crashing_on_startup_twice" + "process_crashing_on_startup_once" ] }, - "run_target_crash_on_startup_five_times": { + "run_target_crash_on_startup_three_times": { "depends_on": [ "control_client_mock", - "process_crashing_on_startup_five_times" + "process_crashing_on_startup_three_times" ] }, - "run_target_crash_on_startup_always": { + "run_target_crash_on_startup_no_retries": { "depends_on": [ "control_client_mock", - "process_crashing_on_startup_always" + "process_crashing_on_startup_no_retries" ], "recovery_action": { "switch_run_target": { diff --git a/tests/integration/crash_on_startup/process_crashing_on_startup_always.cpp b/tests/integration/crash_on_startup/process_crashing_on_startup_always.cpp deleted file mode 100644 index b0df435556..0000000000 --- a/tests/integration/crash_on_startup/process_crashing_on_startup_always.cpp +++ /dev/null @@ -1,19 +0,0 @@ -/******************************************************************************** - * Copyright (c) 2026 Contributors to the Eclipse Foundation - * - * See the NOTICE file(s) distributed with this work for additional - * information regarding copyright ownership. - * - * This program and the accompanying materials are made available under the - * terms of the Apache License Version 2.0 which is available at - * https://www.apache.org/licenses/LICENSE-2.0 - * - * SPDX-License-Identifier: Apache-2.0 - ********************************************************************************/ -#include - -int main() -{ - std::cout << "Process crashing on startup (always)..." << std::endl; - std::abort(); -} diff --git a/tests/utils/test_helper/BUILD b/tests/utils/test_helper/BUILD index d5ef0cde5b..c044cce2cf 100644 --- a/tests/utils/test_helper/BUILD +++ b/tests/utils/test_helper/BUILD @@ -63,3 +63,14 @@ cc_binary( ":test_helper", ], ) + +cc_binary( + name = "process_crashing_on_startup_n_times", + srcs = ["process_crashing_on_startup_n_times.cpp"], + visibility = ["//tests:__subpackages__"], + deps = [ + ":test_helper", + "//score/launch_manager:lifecycle_cc", + "@googletest//:gtest_main", + ], +) diff --git a/tests/integration/crash_on_startup/process_crashing_on_startup_n_times.cpp b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp similarity index 78% rename from tests/integration/crash_on_startup/process_crashing_on_startup_n_times.cpp rename to tests/utils/test_helper/process_crashing_on_startup_n_times.cpp index 230bb37aea..c0af8fe4f9 100644 --- a/tests/integration/crash_on_startup/process_crashing_on_startup_n_times.cpp +++ b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp @@ -12,7 +12,6 @@ ********************************************************************************/ #include #include -#include #include #include #include @@ -23,7 +22,7 @@ /// @brief Default number of times the process crashes before starting up successfully. constexpr int kDefaultCrashesUntilSuccess = 3; -TEST(CrashOnStartup, ProcessCrashingOnStartupTwice) +TEST(CrashOnStartup, ProcessCrashingOnStartupNTimes) { TEST_STEP("Report running") { @@ -52,7 +51,17 @@ void write_crash_count(const std::string_view file_path, const int count) int main(int argc, char** argv) { - // The number of crashes before a successful startup is taken from the command line, defaulting to two. + if (argc > 1 && (std::string_view{argv[1]} == "-h" || std::string_view{argv[1]} == "--help")) + { + std::cout << "Usage: " << argv[0] << " [crashes_until_success]\n" + << "Crashes on startup the given number of times (default " << kDefaultCrashesUntilSuccess + << ") before starting up successfully.\n" + << "The crash count is persisted across restarts in '" << crash_count_file << "'." << std::endl; + return 0; + } + + // The number of crashes before a successful startup is taken from the command line, defaulting to + // kDefaultCrashesUntilSuccess. const int crashes_until_success = (argc > 1) ? std::atoi(argv[1]) : kDefaultCrashesUntilSuccess; const int crash_count = read_crash_count(crash_count_file); From 7bdf9924d415fae540485973c6b2250349d8c146 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 16:45:08 +0200 Subject: [PATCH 06/12] Correct config, write crash count to xml file --- .../crash_on_startup/control_client_mock.cpp | 8 +++---- .../crash_on_startup/crash_on_startup.json | 8 +++---- .../crash_on_startup/crash_on_startup.py | 23 ++++++++++++++----- .../process_crashing_on_startup_n_times.cpp | 22 +++++++++++------- tests/utils/testing_utils/test_results.py | 10 ++++++++ 5 files changed, 49 insertions(+), 22 deletions(-) diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index 82bfed318d..4c7ba40abf 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -28,13 +28,13 @@ TEST(CrashOnStartup, ControlClientMock) score::mw::lifecycle::report_running(); } - // Given a process that crashes on startup once, but is configured to retry twice - so it succeeds - TEST_STEP("Launch process crashing on startup once") + // Given a process that crashes on startup twice, but is configured to retry twice - so it succeeds + TEST_STEP("Launch process crashing on startup twice") { score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_once").Get(stop_token); + auto result = client.ActivateRunTarget("run_target_crash_on_startup_twice").Get(stop_token); // Then, the LM should restart it and eventually succeed - EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_once failed: " + EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_twice failed: " << result.error().Message(); } diff --git a/tests/integration/crash_on_startup/crash_on_startup.json b/tests/integration/crash_on_startup/crash_on_startup.json index 04b35856ae..5776a7f816 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.json +++ b/tests/integration/crash_on_startup/crash_on_startup.json @@ -43,14 +43,14 @@ } } }, - "process_crashing_on_startup_once": { + "process_crashing_on_startup_twice": { "component_properties": { "binary_name": "process_crashing_on_startup_n_times", "application_profile": { "application_type": "Reporting" }, "process_arguments": [ - "1" + "2" ] }, "deployment_config": { @@ -112,10 +112,10 @@ "control_client_mock" ] }, - "run_target_crash_on_startup_once": { + "run_target_crash_on_startup_twice": { "depends_on": [ "control_client_mock", - "process_crashing_on_startup_once" + "process_crashing_on_startup_twice" ] }, "run_target_crash_on_startup_three_times": { diff --git a/tests/integration/crash_on_startup/crash_on_startup.py b/tests/integration/crash_on_startup/crash_on_startup.py index 17f4346428..e0c2ad8359 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.py +++ b/tests/integration/crash_on_startup/crash_on_startup.py @@ -12,7 +12,7 @@ # ******************************************************************************* from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed from tests.utils.testing_utils.setup_test import setup_test -from tests.utils.testing_utils.test_results import assert_test_results +from tests.utils.testing_utils.test_results import assert_test_results, get_testcase_property from attribute_plugin import add_test_properties @@ -26,15 +26,19 @@ test_type="requirements-based", derivation_technique="requirements-analysis", ) -def test_crash_on_startup(target, setup_test, assert_test_results, remote_test_dir): +def test_crash_on_startup( + target, setup_test, assert_test_results, remote_test_dir, test_output_dir +): """ Objective: Verifies that the launch manager correctly handles processes that crash before reporting running. - Case 1: Process crashes before Running state but eventually starts up successfully before the configured number of restart attempts is exceeded. - Expected Behaviour: Process startup successful, RunTarget activation successful + Case 1: Process crashes before Running state but eventually starts up successfully before the configured number of restart attempts is exceeded. + This is verified with two different components: One with the process crashing twice and the other three times before successfully starting up. + The number of restart attempts is configured to be 2 and 3 respectively for these two components. + Expected Behaviour: Process startup successful, run target activation successful - Case 2: Process keeps crashing, exceeding the number of restart attempts. - Expected Behaviour: Process startup fails, LaunchManager executes recovery action. + Case 2: Component has no restart attempts configured, but crashes once. + Expected Behaviour: Process startup fails and therefore run target activation fails. Launch manager executes recovery action which switches to fallback run target. """ config_path = str(remote_test_dir / "etc/crash_on_startup.bin") @@ -51,3 +55,10 @@ def test_crash_on_startup(target, setup_test, assert_test_results, remote_test_d assert_test_results( {"control_client_mock.xml", "process_crashing_on_startup_n_times.xml"} ) + + # The number of crashes before a successful startup is recorded in the report. The last run target to + # start up successfully (run_target_crash_on_startup_three_times) crashes three times before succeeding. + crash_count = get_testcase_property( + test_output_dir / "process_crashing_on_startup_n_times.xml", "crash_count" + ) + assert crash_count == "3", f"Expected 3 crashes before startup, got {crash_count}" diff --git a/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp index c0af8fe4f9..44bbc26239 100644 --- a/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp +++ b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp @@ -22,14 +22,6 @@ /// @brief Default number of times the process crashes before starting up successfully. constexpr int kDefaultCrashesUntilSuccess = 3; -TEST(CrashOnStartup, ProcessCrashingOnStartupNTimes) -{ - TEST_STEP("Report running") - { - score::mw::lifecycle::report_running(); - } -} - /// @brief Reads how many times the process has crashed so far from the crash count file. int read_crash_count(const std::string_view file_path) { @@ -49,6 +41,20 @@ void write_crash_count(const std::string_view file_path, const int count) file << count; } +TEST(CrashOnStartup, ProcessCrashingOnStartupNTimes) +{ + TEST_STEP("Record number of crashes before successful startup") + { + // The crash count file holds the number of crashes that occurred before this successful startup. + RecordProperty("crash_count", read_crash_count(crash_count_file)); + } + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } +} + int main(int argc, char** argv) { if (argc > 1 && (std::string_view{argv[1]} == "-h" || std::string_view{argv[1]} == "--help")) diff --git a/tests/utils/testing_utils/test_results.py b/tests/utils/testing_utils/test_results.py index 3eff9c29b6..8562d77945 100644 --- a/tests/utils/testing_utils/test_results.py +++ b/tests/utils/testing_utils/test_results.py @@ -68,6 +68,16 @@ def get_failing_files(path: Path): return all_files, failing_files +def get_testcase_property(path: Path, name: str) -> str: + """Returns the value of the `name` property recorded on the first testcase of the + given xml result file. gtest emits properties recorded via RecordProperty as + `` elements nested in the ``.""" + root = ElementTree.parse(str(path)).getroot() + prop = root.find(f".//testcase/properties/property[@name='{name}']") + assert prop is not None, f"Property '{name}' not found in {path.name}" + return prop.get("value") + + @pytest.fixture def assert_test_results(target, remote_test_dir, test_output_dir): """Returns a callable that downloads XML results and asserts the expected From fdbf65d6052bd4ff3f6bebbc94149352a0c16f63 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 17:23:20 +0200 Subject: [PATCH 07/12] Avoid duplication --- .../crash_on_startup/control_client_mock.cpp | 55 +++++++------------ 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index 4c7ba40abf..ff5e2419b9 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -28,49 +28,36 @@ TEST(CrashOnStartup, ControlClientMock) score::mw::lifecycle::report_running(); } - // Given a process that crashes on startup twice, but is configured to retry twice - so it succeeds - TEST_STEP("Launch process crashing on startup twice") + // Given a process that crashes on startup n times, but is configured to retry n times - so it eventually + // succeeds. The behaviour is identical for the different crash counts, so it is parameterized over the + // corresponding run targets. + for (const std::string_view run_target : + {"run_target_crash_on_startup_two_times", "run_target_crash_on_startup_three_times"}) { - score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_twice").Get(stop_token); - // Then, the LM should restart it and eventually succeed - EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_twice failed: " - << result.error().Message(); - } - - TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully") - { - EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; - } + TEST_STEP(std::string{"Launch "} + std::string{run_target}) + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget(run_target).Get(stop_token); + // Then, the LM should restart it and eventually succeed + EXPECT_TRUE(result.has_value()) << "Activating " << run_target << " failed: " << result.error().Message(); + } - EXPECT_TRUE(std::filesystem::remove(crash_count_file)) - << "Count file must be removed successfully, before reused in the next run target"; + TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully") + { + EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; + } - // Given a process that crashes on startup three times, but is configured to retry three times - so it still - // succeeds - TEST_STEP("Launch process crashing on startup three times") - { - score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_three_times").Get(stop_token); - // Then, the LM should restart it and eventually succeed - EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_three_times failed: " - << result.error().Message(); + EXPECT_TRUE(std::filesystem::remove(crash_count_file)) + << "Count file must be removed successfully, before reused in the next run target"; } - TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully") - { - EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; - } - - EXPECT_TRUE(std::filesystem::remove(crash_count_file)) - << "Count file must be removed successfully, before reused in the next run target"; - // Given a process that crashes on startup but is not allowed to retry (number_of_attempts=0) TEST_STEP("Attempt to launch process crashing on startup without retries") { score::cpp::stop_token stop_token; - auto result = client.ActivateRunTarget("run_target_crash_on_startup_no_retries").Get(stop_token); - EXPECT_FALSE(result.has_value()) << "Expected run_target_crash_on_startup_no_retries activation to fail"; + auto result = client.ActivateRunTarget("run_target_crash_on_startup_once_but_no_retries").Get(stop_token); + EXPECT_FALSE(result.has_value()) + << "Expected run_target_crash_on_startup_once_but_no_retries activation to fail"; } // Limitation: we cannot wait for the transition to fallback to complete sleep(1); From 293fc77b4b2b511f8e92fa24d77a74ce4d281fd7 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 17:24:56 +0200 Subject: [PATCH 08/12] Rename run targets --- tests/integration/crash_on_startup/crash_on_startup.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/crash_on_startup/crash_on_startup.json b/tests/integration/crash_on_startup/crash_on_startup.json index 5776a7f816..124aed307a 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.json +++ b/tests/integration/crash_on_startup/crash_on_startup.json @@ -112,7 +112,7 @@ "control_client_mock" ] }, - "run_target_crash_on_startup_twice": { + "run_target_crash_on_startup_two_times": { "depends_on": [ "control_client_mock", "process_crashing_on_startup_twice" @@ -124,7 +124,7 @@ "process_crashing_on_startup_three_times" ] }, - "run_target_crash_on_startup_no_retries": { + "run_target_crash_on_startup_once_but_no_retries": { "depends_on": [ "control_client_mock", "process_crashing_on_startup_no_retries" From bfda214124f07076d8b32eea4287225209005987 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 17:29:20 +0200 Subject: [PATCH 09/12] Verify count of crashs in python --- .../crash_on_startup/crash_on_startup.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/integration/crash_on_startup/crash_on_startup.py b/tests/integration/crash_on_startup/crash_on_startup.py index e0c2ad8359..fa8b5a40e2 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.py +++ b/tests/integration/crash_on_startup/crash_on_startup.py @@ -52,13 +52,17 @@ def test_crash_on_startup( timeout_s=10.0, ) - assert_test_results( - {"control_client_mock.xml", "process_crashing_on_startup_n_times.xml"} - ) + # Each crashing process writes its own report file named after the number of times it crashes, so the + # reports of the different run targets no longer overwrite each other. The process crashing once is not + # allowed to retry, but still writes its report before crashing. + crash_report_names = { + n: f"process_crashing_on_startup_n_times_n_equals_{n}.xml" for n in (1, 2, 3) + } + assert_test_results({"control_client_mock.xml", *crash_report_names.values()}) - # The number of crashes before a successful startup is recorded in the report. The last run target to - # start up successfully (run_target_crash_on_startup_three_times) crashes three times before succeeding. - crash_count = get_testcase_property( - test_output_dir / "process_crashing_on_startup_n_times.xml", "crash_count" - ) - assert crash_count == "3", f"Expected 3 crashes before startup, got {crash_count}" + # The number of crashes is recorded in each report and must match the configured crash count. + for n, report_name in crash_report_names.items(): + crash_count = get_testcase_property(test_output_dir / report_name, "crash_count") + assert crash_count == str(n), ( + f"Expected {n} crashes in {report_name}, got {crash_count}" + ) From 12cb0bad4bd283748179d0de24d6d02c33f8100b Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 17:33:47 +0200 Subject: [PATCH 10/12] Adapt test process to write crash count in report --- .../process_crashing_on_startup_n_times.cpp | 33 ++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp index 44bbc26239..2c1d85783b 100644 --- a/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp +++ b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp @@ -12,8 +12,10 @@ ********************************************************************************/ #include #include +#include #include #include +#include #include #include "tests/utils/test_helper/test_helper.hpp" @@ -41,14 +43,26 @@ void write_crash_count(const std::string_view file_path, const int count) file << count; } +/// @brief Whether this invocation reaches the running state. False when simulating a crash on startup, in +/// which case the process must not report running. +bool g_startup_successful = false; + TEST(CrashOnStartup, ProcessCrashingOnStartupNTimes) { - TEST_STEP("Record number of crashes before successful startup") + TEST_STEP("Record number of crashes so far") { - // The crash count file holds the number of crashes that occurred before this successful startup. + // The crash count file holds the number of crashes that occurred, including the crash simulated by + // this invocation (if any). RecordProperty("crash_count", read_crash_count(crash_count_file)); } + if (!g_startup_successful) + { + // This invocation simulates a crash before reaching the running state, so it must not report + // running. The process aborts in main() once this report has been written. + return; + } + TEST_STEP("Report running") { score::mw::lifecycle::report_running(); @@ -62,7 +76,9 @@ int main(int argc, char** argv) std::cout << "Usage: " << argv[0] << " [crashes_until_success]\n" << "Crashes on startup the given number of times (default " << kDefaultCrashesUntilSuccess << ") before starting up successfully.\n" - << "The crash count is persisted across restarts in '" << crash_count_file << "'." << std::endl; + << "The crash count is persisted across restarts in '" << crash_count_file << "'.\n" + << "Each configuration writes its own report, e.g. '_n_equals_.xml'." + << std::endl; return 0; } @@ -70,15 +86,24 @@ int main(int argc, char** argv) // kDefaultCrashesUntilSuccess. const int crashes_until_success = (argc > 1) ? std::atoi(argv[1]) : kDefaultCrashesUntilSuccess; + // Each configuration writes its own report file, e.g. process_crashing_on_startup_n_times_n_equals_2.xml, + // so that the report of a process crashing n times is not overwritten by one crashing a different number of times. + const std::string report_path = + std::filesystem::path{__FILE__}.stem().string() + "_n_equals_" + std::to_string(crashes_until_success); + const int crash_count = read_crash_count(crash_count_file); if (crash_count < crashes_until_success) { std::cout << "Process crashing on startup (" << (crash_count + 1) << "/" << crashes_until_success << ")..." << std::endl; write_crash_count(crash_count_file, crash_count + 1); + // Write the report before crashing, so that even a process that is never allowed to start up + // successfully (e.g. no retries) still produces its report. + TestRunner(report_path, TerminationBehavior::kContinue).RunTests(); std::abort(); } std::cout << "Process starting successfully..." << std::endl; - return TestRunner(__FILE__).RunTests(); + g_startup_successful = true; + return TestRunner(report_path).RunTests(); } From f1efb7f0b3ef781c54552ec2d8835f0e4240b8a6 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 17:51:51 +0200 Subject: [PATCH 11/12] Append param to crash_count file --- .../crash_on_startup/control_client_mock.cpp | 9 ++++----- .../process_crashing_on_startup_n_times.cpp | 19 +++++++++++++------ tests/utils/test_helper/test_helper.hpp | 9 ++++++++- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index ff5e2419b9..a0e530da10 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -21,7 +21,8 @@ TEST(CrashOnStartup, ControlClientMock) { score::mw::lifecycle::ControlClient client; - ASSERT_TRUE(check_clean({crash_count_file, test_end_location, fallback_file})); + ASSERT_TRUE(check_clean( + {crashCountPath(1), crashCountPath(2), crashCountPath(3), test_end_location, fallback_file})); TEST_STEP("Report running") { @@ -30,7 +31,8 @@ TEST(CrashOnStartup, ControlClientMock) // Given a process that crashes on startup n times, but is configured to retry n times - so it eventually // succeeds. The behaviour is identical for the different crash counts, so it is parameterized over the - // corresponding run targets. + // corresponding run targets. Each run target's process persists its crash count in its own file, so the + // run targets do not interfere with each other. for (const std::string_view run_target : {"run_target_crash_on_startup_two_times", "run_target_crash_on_startup_three_times"}) { @@ -46,9 +48,6 @@ TEST(CrashOnStartup, ControlClientMock) { EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet"; } - - EXPECT_TRUE(std::filesystem::remove(crash_count_file)) - << "Count file must be removed successfully, before reused in the next run target"; } // Given a process that crashes on startup but is not allowed to retry (number_of_attempts=0) diff --git a/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp index 2c1d85783b..c88dcafc04 100644 --- a/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp +++ b/tests/utils/test_helper/process_crashing_on_startup_n_times.cpp @@ -21,6 +21,8 @@ #include "tests/utils/test_helper/test_helper.hpp" #include +namespace +{ /// @brief Default number of times the process crashes before starting up successfully. constexpr int kDefaultCrashesUntilSuccess = 3; @@ -47,13 +49,18 @@ void write_crash_count(const std::string_view file_path, const int count) /// which case the process must not report running. bool g_startup_successful = false; +/// @brief Path to this invocation's crash count file, set in main() from the configured crash count. Every +/// configured crash count gets its own file, so that concurrently configured processes do not share state. +std::string g_crash_count_path; +} // namespace + TEST(CrashOnStartup, ProcessCrashingOnStartupNTimes) { TEST_STEP("Record number of crashes so far") { // The crash count file holds the number of crashes that occurred, including the crash simulated by // this invocation (if any). - RecordProperty("crash_count", read_crash_count(crash_count_file)); + RecordProperty("crash_count", read_crash_count(g_crash_count_path)); } if (!g_startup_successful) @@ -76,9 +83,8 @@ int main(int argc, char** argv) std::cout << "Usage: " << argv[0] << " [crashes_until_success]\n" << "Crashes on startup the given number of times (default " << kDefaultCrashesUntilSuccess << ") before starting up successfully.\n" - << "The crash count is persisted across restarts in '" << crash_count_file << "'.\n" - << "Each configuration writes its own report, e.g. '_n_equals_.xml'." - << std::endl; + << "Each configuration persists its crash count and writes its own report in files suffixed " + << "with '_'." << std::endl; return 0; } @@ -90,13 +96,14 @@ int main(int argc, char** argv) // so that the report of a process crashing n times is not overwritten by one crashing a different number of times. const std::string report_path = std::filesystem::path{__FILE__}.stem().string() + "_n_equals_" + std::to_string(crashes_until_success); + g_crash_count_path = crashCountPath(crashes_until_success); - const int crash_count = read_crash_count(crash_count_file); + const int crash_count = read_crash_count(g_crash_count_path); if (crash_count < crashes_until_success) { std::cout << "Process crashing on startup (" << (crash_count + 1) << "/" << crashes_until_success << ")..." << std::endl; - write_crash_count(crash_count_file, crash_count + 1); + write_crash_count(g_crash_count_path, crash_count + 1); // Write the report before crashing, so that even a process that is never allowed to start up // successfully (e.g. no retries) still produces its report. TestRunner(report_path, TerminationBehavior::kContinue).RunTests(); diff --git a/tests/utils/test_helper/test_helper.hpp b/tests/utils/test_helper/test_helper.hpp index eb57ccc75c..569ec7f1b6 100644 --- a/tests/utils/test_helper/test_helper.hpp +++ b/tests/utils/test_helper/test_helper.hpp @@ -46,9 +46,16 @@ inline testing::AssertionResult touch_file(const std::string_view file_path) /// @brief Location to store a file signalling that the fallback state has been reached. constexpr std::string_view fallback_file = "fallback_reached"; -/// @brief Location to store the number of times a process has crashed so far - used to test ready recovery action +/// @brief Prefix of the file storing the number of times a process has crashed so far - used to test ready +/// recovery action constexpr std::string_view crash_count_file = "crash_count"; +/// @return File path to store the crash count for a process configured to crash `crashes_until_success` times +inline std::string crashCountPath(const int crashes_until_success) +{ + return std::string{crash_count_file} + "_" + std::to_string(crashes_until_success); +} + /// @brief Where to store the test_end signal file. This must be kept consistent with where the test framework /// searches for files. constexpr std::string_view test_end_location = "../test_end"; From ce1fc6df115845b2615dbe9d48b23de34a03cc07 Mon Sep 17 00:00:00 2001 From: Timo Steuerwald Date: Mon, 10 Aug 2026 17:53:29 +0200 Subject: [PATCH 12/12] Fix formating --- .../crash_on_startup/control_client_mock.cpp | 4 ++-- .../crash_on_startup/crash_on_startup.py | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/integration/crash_on_startup/control_client_mock.cpp b/tests/integration/crash_on_startup/control_client_mock.cpp index a0e530da10..f8a5e3089f 100644 --- a/tests/integration/crash_on_startup/control_client_mock.cpp +++ b/tests/integration/crash_on_startup/control_client_mock.cpp @@ -21,8 +21,8 @@ TEST(CrashOnStartup, ControlClientMock) { score::mw::lifecycle::ControlClient client; - ASSERT_TRUE(check_clean( - {crashCountPath(1), crashCountPath(2), crashCountPath(3), test_end_location, fallback_file})); + ASSERT_TRUE( + check_clean({crashCountPath(1), crashCountPath(2), crashCountPath(3), test_end_location, fallback_file})); TEST_STEP("Report running") { diff --git a/tests/integration/crash_on_startup/crash_on_startup.py b/tests/integration/crash_on_startup/crash_on_startup.py index fa8b5a40e2..2a074bed76 100644 --- a/tests/integration/crash_on_startup/crash_on_startup.py +++ b/tests/integration/crash_on_startup/crash_on_startup.py @@ -12,7 +12,10 @@ # ******************************************************************************* from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed from tests.utils.testing_utils.setup_test import setup_test -from tests.utils.testing_utils.test_results import assert_test_results, get_testcase_property +from tests.utils.testing_utils.test_results import ( + assert_test_results, + get_testcase_property, +) from attribute_plugin import add_test_properties @@ -32,8 +35,8 @@ def test_crash_on_startup( """ Objective: Verifies that the launch manager correctly handles processes that crash before reporting running. - Case 1: Process crashes before Running state but eventually starts up successfully before the configured number of restart attempts is exceeded. - This is verified with two different components: One with the process crashing twice and the other three times before successfully starting up. + Case 1: Process crashes before Running state but eventually starts up successfully before the configured number of restart attempts is exceeded. + This is verified with two different components: One with the process crashing twice and the other three times before successfully starting up. The number of restart attempts is configured to be 2 and 3 respectively for these two components. Expected Behaviour: Process startup successful, run target activation successful @@ -62,7 +65,9 @@ def test_crash_on_startup( # The number of crashes is recorded in each report and must match the configured crash count. for n, report_name in crash_report_names.items(): - crash_count = get_testcase_property(test_output_dir / report_name, "crash_count") + crash_count = get_testcase_property( + test_output_dir / report_name, "crash_count" + ) assert crash_count == str(n), ( f"Expected {n} crashes in {report_name}, got {crash_count}" )