From 243d532811094f34b4b31fd59f0cf323e1131d68 Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:58:46 +0100 Subject: [PATCH 1/8] Fix test timeout --- .../testing_utils/run_until_file_deployed.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/utils/testing_utils/run_until_file_deployed.py b/tests/utils/testing_utils/run_until_file_deployed.py index a5ba530a88..1837eadbbf 100644 --- a/tests/utils/testing_utils/run_until_file_deployed.py +++ b/tests/utils/testing_utils/run_until_file_deployed.py @@ -62,21 +62,7 @@ def run_until_file_deployed( exit_code, _ = target.execute(f"test -f {file_path}") if exit_code == 0: - # Kill the entire process group so that children (e.g. the actual - # daemon binary launched under fakeroot) receive SIGTERM and can - # run their cleanup code before exiting. - kill_cmd = f"kill -TERM {proc.pid()}" - res, _ = target.execute(kill_cmd) - assert res == 0, "Couldn't kill lcm with SIGTERM" - try: - # wait() raises RuntimeError if the process is still running after - # stop_timeout_s. - exit_code = proc.wait(timeout_s=stop_timeout_s) - except RuntimeError as exc: - proc.stop() # escalate to SIGKILL so we don't leak the process - assert False, ( - f"Process '{binary_path}' still running {stop_timeout_s}s after SIGTERM: {exc}" - ) + exit_code = proc.stop() assert exit_code == 0, ( f"LCM did not exit cleanly, it died with code {exit_code}" ) From 5f32923e3215309dbff0a0d07aca4d14bb467d66 Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:05:44 +0100 Subject: [PATCH 2/8] Add new test for failure --- .../fallback_to_same_target_restarts/BUILD | 47 ++++++++ .../control_client_mock.cpp | 70 +++++++++++ .../fallback_to_same_target_restarts.json | 109 ++++++++++++++++++ .../fallback_to_same_target_restarts.py | 48 ++++++++ .../process_crashing_once.cpp | 50 ++++++++ 5 files changed, 324 insertions(+) create mode 100644 tests/integration/fallback_to_same_target_restarts/BUILD create mode 100644 tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp create mode 100644 tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.json create mode 100644 tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py create mode 100644 tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp diff --git a/tests/integration/fallback_to_same_target_restarts/BUILD b/tests/integration/fallback_to_same_target_restarts/BUILD new file mode 100644 index 0000000000..eb8e8a00c6 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/BUILD @@ -0,0 +1,47 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_binary( + name = "control_client_mock", + srcs = ["control_client_mock.cpp"], + deps = [ + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +cc_binary( + name = "process_crashing_once", + srcs = ["process_crashing_once.cpp"], + deps = [ + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "fallback_to_same_target_restarts", + srcs = ["fallback_to_same_target_restarts.py"], + binaries = [ + ":config", + ":control_client_mock", + ":process_crashing_once", + "//score/launch_manager" + ], + config = ":fallback_to_same_target_restarts.json", +) diff --git a/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp b/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp new file mode 100644 index 0000000000..f834a113f3 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp @@ -0,0 +1,70 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include +#include +#include +#include + +// Given a correct configuration with: +// - An initial Run Target named "Startup" containing "control_client_mock" +// - A Run Target named "run_target_crashing_app_on_runtime" containing "control_client_mock" and +// "component_crashing_once" + +TEST(FallbackToSameTargetRestarts, ControlClientMock) +{ + score::mw::lifecycle::ControlClient client; + + const std::string_view process_file = "process_started_normally"; + + ASSERT_TRUE(check_clean({test_end_location, process_file})); + // Establish communication with launch manager + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Start crashing process") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_crashing_app_on_runtime").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating target run_target_crashing_app_on_runtime failed: " + << result.error().Message(); + } + // When the process crashes, wait for the fallback to be activated. + // Use polling instead of a fixed sleep so the test is robust under slow builds (e.g. TSan). + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!std::filesystem::exists(process_file) && std::chrono::steady_clock::now() < deadline) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } + // Then + TEST_STEP("Verify process was restarted") + { + EXPECT_TRUE(std::filesystem::exists(process_file)) << "Process did not restart successfully"; + } + TEST_STEP("Activate RunTarget Off") + { + client.ActivateRunTarget("Off"); + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kWait, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.json b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.json new file mode 100644 index 0000000000..173ee643c8 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.json @@ -0,0 +1,109 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/fallback_to_same_target_restarts", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "control_client_mock": { + "component_properties": { + "binary_name": "control_client_mock", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_mock" + } + } + }, + "component_crashing_once": { + "component_properties": { + "binary_name": "process_crashing_once", + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": true + } + }, + "deployment_config": { + "environmental_variables": { + "PROCESSIDENTIFIER": "component_crashing_once" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "control_client_mock" + ] + }, + "run_target_crashing_app_on_runtime": { + "depends_on": [ + "control_client_mock", + "component_crashing_once" + ], + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + } + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [ + "control_client_mock", + "component_crashing_once" + ] + } +} diff --git a/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py new file mode 100644 index 0000000000..c9e9f72479 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py @@ -0,0 +1,48 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + test_type="interface-test", + derivation_technique="error-guessing", +) +def test_fallback_to_same_target_restarts( + target, setup_test, assert_test_results, remote_test_dir +): + """ + Objective: Verifies that the launch manager correctly restarts a crashed process if it is active in the fallback state. + + A process crashes after run target activation completes and proceeds normally the second time it is launched. + Expected Behaviour: The process is relaunched and completes normally. + """ + + # launch manager will simply ignore the arguments if run with --//config:use_new_configuration=False. + # the old configuration will be used, which is the default behavior. + # The new configuration will be used if run with --//config:use_new_configuration=True + new_config_path = str(remote_test_dir / "etc/process_crash_monitoring.bin") + + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=remote_test_dir.parent / "test_end", + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=4.0, + ) + + assert_test_results({"control_client_mock.xml", "process_crashing_once.xml"}) diff --git a/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp b/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp new file mode 100644 index 0000000000..d236e4d8a5 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp @@ -0,0 +1,50 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include + +TEST(FallbackToSameTargetRestarts, CrashingProcess) +{ + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + // Limitation: we can't wait for run target activation to complete + sleep(1); + + TEST_STEP("Crash if we haven't crashed yet") + { + const std::string_view crash_file = "process_crashed"; + + if (!std::filesystem::exists(crash_file)) + { + std::cout << "Process crashing..." << std::endl; + if (!touch_file(crash_file)) + { + std::cout << "Failed to deploy marker file!" << std::endl; + } + std::abort(); + } + + ASSERT_TRUE(touch_file("process_started_normally")); + std::cout << "Process finishing normally" << std::endl; + } +} + +int main() +{ + TestRunner(__FILE__, TerminationBehavior::kContinue).RunTests(); +} From 9feb60b23728ed3fd937c995c5ac72608c319268 Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:08:12 +0100 Subject: [PATCH 3/8] Fix incorrect behaviour --- .../src/daemon/src/process_group_manager/details/graph.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp index 3e1f35b7c0..26cc16f405 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp @@ -428,6 +428,11 @@ void Graph::handleComponentEvent(const ComponentEvent& event) // This is always an error after ready - an unexpected termination before ready is an activation failure const auto error = IComponent::ComponentError::kErrorAfterReady; abort(1, error); + + // Need to clean up any leftover resources + IComponent& failingComponent = componentOf(nodes_[data.node_index]); + static_cast(failingComponent.deactivate({})); + if (jobs_in_progress_ == 0) { handleNonTransitionExecution(getState()); From deeb3007a299f30ca9405440695d03dc7883ed8f Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Wed, 5 Aug 2026 17:23:19 +0100 Subject: [PATCH 4/8] Fix format --- tests/integration/fallback_to_same_target_restarts/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/fallback_to_same_target_restarts/BUILD b/tests/integration/fallback_to_same_target_restarts/BUILD index eb8e8a00c6..e2faa68258 100644 --- a/tests/integration/fallback_to_same_target_restarts/BUILD +++ b/tests/integration/fallback_to_same_target_restarts/BUILD @@ -41,7 +41,7 @@ integration_test( ":config", ":control_client_mock", ":process_crashing_once", - "//score/launch_manager" + "//score/launch_manager", ], config = ":fallback_to_same_target_restarts.json", ) From 8141863e10cdc277ca2a541739cd84d163fe13fa Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:00:58 +0100 Subject: [PATCH 5/8] Remove unused parameter --- tests/utils/testing_utils/run_until_file_deployed.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/utils/testing_utils/run_until_file_deployed.py b/tests/utils/testing_utils/run_until_file_deployed.py index 1837eadbbf..d765e11038 100644 --- a/tests/utils/testing_utils/run_until_file_deployed.py +++ b/tests/utils/testing_utils/run_until_file_deployed.py @@ -25,7 +25,6 @@ def run_until_file_deployed( file_path: str, timeout_s: float = 30.0, poll_interval_s: float = 0.5, - stop_timeout_s: float = 3.0, args=None, cwd: str = "/", ) -> AsyncProcess: @@ -36,8 +35,6 @@ def run_until_file_deployed( :param file_path: path of the file to wait for on the target. :param timeout_s: maximum seconds to wait for the file (default: 30). :param poll_interval_s: seconds between file checks (default: 0.5). - :param stop_timeout_s: maximum seconds to wait for the process to terminate - after SIGTERM (default: 3). :param args: optional list of arguments to pass to the binary. :param cwd: working directory on the target (default: "/"). :return: the stopped :class:`AsyncProcess` handle. From 2215843db6e998ccac86108916260e38fbf4f760 Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Fri, 7 Aug 2026 09:08:30 +0100 Subject: [PATCH 6/8] Fix UT, update comment --- .../process_group_manager/details/graph.cpp | 2 +- .../details/graph_UT.cpp | 20 +++++++++++++++++-- .../fallback_to_same_target_restarts.py | 5 +---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp index 26cc16f405..8ebe1fa313 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp @@ -428,7 +428,7 @@ void Graph::handleComponentEvent(const ComponentEvent& event) // This is always an error after ready - an unexpected termination before ready is an activation failure const auto error = IComponent::ComponentError::kErrorAfterReady; abort(1, error); - + // Need to clean up any leftover resources IComponent& failingComponent = componentOf(nodes_[data.node_index]); static_cast(failingComponent.deactivate({})); diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp index 6a89ee0061..3f886909e8 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp @@ -440,11 +440,19 @@ TEST_F(GraphHandleComponentEventTest, unexpectedTerminationDuringSuccess) { RecordProperty( "Description", - "Test that an unexpected termination after a successful transition causes the graph to enter an undefined " - "state"); + "Test that an unexpected termination after a successful transition causes the graph to deactivate the " + "component and enter an undefined state"); completeTransition(state_name(run_target_name(0))); + const auto component = graph_.getProcessInfoNode(0); + EXPECT_CALL(process_interface_, requestTermination) + .WillOnce(DoAll( + InvokeWithoutArgs([component] { + static_cast(component->tryHandleTermination(134)); + }), + Return(osal::OsalReturnType::kSuccess))); + graph_.handleComponentEvent(UnexpectedTermination{0}); EXPECT_EQ(graph_.getState(), GraphState::kUndefinedState); @@ -464,6 +472,14 @@ TEST_F(GraphHandleComponentEventTest, unexpectedTerminationDuringTransition) const auto component_index = first_job.value()->component.get().getIndex(); graph_.handleComponentEvent(ActivationSuccessful{component_index}); + const auto component = graph_.getProcessInfoNode(component_index); + EXPECT_CALL(process_interface_, requestTermination) + .WillOnce(DoAll( + InvokeWithoutArgs([component] { + static_cast(component->tryHandleTermination(134)); + }), + Return(osal::OsalReturnType::kSuccess))); + // The active component then crashes graph_.handleComponentEvent(UnexpectedTermination{component_index}); diff --git a/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py index c9e9f72479..c2716cd011 100644 --- a/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py +++ b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py @@ -31,10 +31,7 @@ def test_fallback_to_same_target_restarts( Expected Behaviour: The process is relaunched and completes normally. """ - # launch manager will simply ignore the arguments if run with --//config:use_new_configuration=False. - # the old configuration will be used, which is the default behavior. - # The new configuration will be used if run with --//config:use_new_configuration=True - new_config_path = str(remote_test_dir / "etc/process_crash_monitoring.bin") + new_config_path = str(remote_test_dir / "etc/fallback_to_same_target_restarts.bin") run_until_file_deployed( target=target, From 0178e0940e39d8660c9888a11f142496a88f9849 Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:21:02 +0100 Subject: [PATCH 7/8] Add new test --- .../crash_ignores_dependents/BUILD | 47 +++++++++ .../crash_ignores_dependents.json | 96 +++++++++++++++++++ .../crash_ignores_dependents.py | 45 +++++++++ .../process_crashing_once.cpp | 50 ++++++++++ .../crash_ignores_dependents/test_process.cpp | 57 +++++++++++ .../control_client_mock.cpp | 1 - tests/utils/test_helper/test_helper.hpp | 2 +- 7 files changed, 296 insertions(+), 2 deletions(-) create mode 100644 tests/integration/crash_ignores_dependents/BUILD create mode 100644 tests/integration/crash_ignores_dependents/crash_ignores_dependents.json create mode 100644 tests/integration/crash_ignores_dependents/crash_ignores_dependents.py create mode 100644 tests/integration/crash_ignores_dependents/process_crashing_once.cpp create mode 100644 tests/integration/crash_ignores_dependents/test_process.cpp diff --git a/tests/integration/crash_ignores_dependents/BUILD b/tests/integration/crash_ignores_dependents/BUILD new file mode 100644 index 0000000000..039993802c --- /dev/null +++ b/tests/integration/crash_ignores_dependents/BUILD @@ -0,0 +1,47 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_binary( + name = "test_process", + srcs = ["test_process.cpp"], + deps = [ + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +cc_binary( + name = "process_crashing_once", + srcs = ["process_crashing_once.cpp"], + deps = [ + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "crash_ignores_dependents", + srcs = ["crash_ignores_dependents.py"], + binaries = [ + ":config", + ":test_process", + ":process_crashing_once", + "//score/launch_manager", + ], + config = ":crash_ignores_dependents.json", +) diff --git a/tests/integration/crash_ignores_dependents/crash_ignores_dependents.json b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.json new file mode 100644 index 0000000000..1cbf6f019c --- /dev/null +++ b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/crash_ignores_dependents", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "test_process": { + "component_properties": { + "binary_name": "test_process", + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": true, + "alive_supervision": { + "min_indications": 0 + } + }, + "depends_on": ["component_crashing_once"] + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "test_process" + } + } + }, + "component_crashing_once": { + "component_properties": { + "binary_name": "process_crashing_once", + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": true + } + }, + "deployment_config": { + "environmental_variables": { + "PROCESSIDENTIFIER": "component_crashing_once" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "test_process" + ] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [ + "test_process" + ] + } +} diff --git a/tests/integration/crash_ignores_dependents/crash_ignores_dependents.py b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.py new file mode 100644 index 0000000000..0c55c3d56c --- /dev/null +++ b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.py @@ -0,0 +1,45 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + test_type="interface-test", + derivation_technique="design-analysis", +) +def test_crash_ignores_dependents( + target, setup_test, assert_test_results, remote_test_dir +): + """ + Objective: Verifies that the launch manager does not restart a process if a process it depends on crashes. + + A process crashes after run target activation completes and proceeds normally the second time it is launched. + Expected Behaviour: The process that depends on it is not interrupted or restarted. + """ + + new_config_path = str(remote_test_dir / "etc/crash_ignores_dependents.bin") + + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=remote_test_dir.parent / "test_end", + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=6.0, + ) + + assert_test_results({"test_process.xml", "process_crashing_once.xml"}) diff --git a/tests/integration/crash_ignores_dependents/process_crashing_once.cpp b/tests/integration/crash_ignores_dependents/process_crashing_once.cpp new file mode 100644 index 0000000000..3c1bbc93bc --- /dev/null +++ b/tests/integration/crash_ignores_dependents/process_crashing_once.cpp @@ -0,0 +1,50 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include + +TEST(CrashIgnoresDependents, CrashingProcess) +{ + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + // Limitation: we can't wait for run target activation to complete + sleep(1); + + TEST_STEP("Crash if we haven't crashed yet") + { + const std::string_view crash_file = "process_crashed"; + + if (!std::filesystem::exists(crash_file)) + { + std::cout << "Process crashing..." << std::endl; + if (!touch_file(crash_file)) + { + std::cout << "Failed to deploy marker file!" << std::endl; + } + std::abort(); + } + + ASSERT_TRUE(touch_file("process_started_normally")); + std::cout << "Process finishing normally" << std::endl; + } +} + +int main() +{ + TestRunner(__FILE__, TerminationBehavior::kContinue).RunTests(); +} diff --git a/tests/integration/crash_ignores_dependents/test_process.cpp b/tests/integration/crash_ignores_dependents/test_process.cpp new file mode 100644 index 0000000000..57a276c13a --- /dev/null +++ b/tests/integration/crash_ignores_dependents/test_process.cpp @@ -0,0 +1,57 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include +#include +#include + +TEST(CrashIgnoresDependents, TestProcess) +{ + const std::string_view started_file = "test_process_started"; + const std::string_view crash_file = "process_crashed"; + + TEST_STEP("Check this is the first start") + { + ASSERT_TRUE(check_clean({test_end_location, started_file, crash_file})) + << "Process was started more than once!"; + + ASSERT_TRUE(touch_file(started_file)) << "Failed to deploy file!"; + } + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + // Wait for the crashing process to deploy its first file + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!std::filesystem::exists(crash_file) && std::chrono::steady_clock::now() < deadline) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } + // Then + TEST_STEP("Verify process crashed while this was running") + { + EXPECT_TRUE(std::filesystem::exists(crash_file)) << "Process did not actually crash"; + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kContinue, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp b/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp index f834a113f3..a00645d617 100644 --- a/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp +++ b/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp @@ -45,7 +45,6 @@ TEST(FallbackToSameTargetRestarts, ControlClientMock) << result.error().Message(); } // When the process crashes, wait for the fallback to be activated. - // Use polling instead of a fixed sleep so the test is robust under slow builds (e.g. TSan). { const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); while (!std::filesystem::exists(process_file) && std::chrono::steady_clock::now() < deadline) diff --git a/tests/utils/test_helper/test_helper.hpp b/tests/utils/test_helper/test_helper.hpp index b0ba2a732c..ff7c397478 100644 --- a/tests/utils/test_helper/test_helper.hpp +++ b/tests/utils/test_helper/test_helper.hpp @@ -19,7 +19,7 @@ #include /// @return File path to an xml adjacent to the input file path -std::string xmlPath(const std::string_view file) +inline std::string xmlPath(const std::string_view file) { return std::filesystem::path{file}.filename().stem().string() + ".xml"; } From 2c37fcfb06c42ebdddc5ebc9d6ca8491610389c2 Mon Sep 17 00:00:00 2001 From: William Roebuck <244554584+WilliamRoebuck@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:20:23 +0100 Subject: [PATCH 8/8] Fix tsan failure --- .../crash_ignores_dependents/process_crashing_once.cpp | 2 +- .../fallback_to_same_target_restarts/process_crashing_once.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/crash_ignores_dependents/process_crashing_once.cpp b/tests/integration/crash_ignores_dependents/process_crashing_once.cpp index 3c1bbc93bc..e3ff6f1826 100644 --- a/tests/integration/crash_ignores_dependents/process_crashing_once.cpp +++ b/tests/integration/crash_ignores_dependents/process_crashing_once.cpp @@ -36,7 +36,7 @@ TEST(CrashIgnoresDependents, CrashingProcess) { std::cout << "Failed to deploy marker file!" << std::endl; } - std::abort(); + exit(1); } ASSERT_TRUE(touch_file("process_started_normally")); diff --git a/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp b/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp index d236e4d8a5..d9b8e14245 100644 --- a/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp +++ b/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp @@ -36,7 +36,7 @@ TEST(FallbackToSameTargetRestarts, CrashingProcess) { std::cout << "Failed to deploy marker file!" << std::endl; } - std::abort(); + exit(1); } ASSERT_TRUE(touch_file("process_started_normally"));