diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp index 3e1f35b7c0..8ebe1fa313 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph.cpp @@ -428,6 +428,11 @@ void Graph::handleComponentEvent(const ComponentEvent& event) // This is always an error after ready - an unexpected termination before ready is an activation failure const auto error = IComponent::ComponentError::kErrorAfterReady; abort(1, error); + + // Need to clean up any leftover resources + IComponent& failingComponent = componentOf(nodes_[data.node_index]); + static_cast(failingComponent.deactivate({})); + if (jobs_in_progress_ == 0) { handleNonTransitionExecution(getState()); diff --git a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp index 6a89ee0061..3f886909e8 100644 --- a/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp +++ b/score/launch_manager/src/daemon/src/process_group_manager/details/graph_UT.cpp @@ -440,11 +440,19 @@ TEST_F(GraphHandleComponentEventTest, unexpectedTerminationDuringSuccess) { RecordProperty( "Description", - "Test that an unexpected termination after a successful transition causes the graph to enter an undefined " - "state"); + "Test that an unexpected termination after a successful transition causes the graph to deactivate the " + "component and enter an undefined state"); completeTransition(state_name(run_target_name(0))); + const auto component = graph_.getProcessInfoNode(0); + EXPECT_CALL(process_interface_, requestTermination) + .WillOnce(DoAll( + InvokeWithoutArgs([component] { + static_cast(component->tryHandleTermination(134)); + }), + Return(osal::OsalReturnType::kSuccess))); + graph_.handleComponentEvent(UnexpectedTermination{0}); EXPECT_EQ(graph_.getState(), GraphState::kUndefinedState); @@ -464,6 +472,14 @@ TEST_F(GraphHandleComponentEventTest, unexpectedTerminationDuringTransition) const auto component_index = first_job.value()->component.get().getIndex(); graph_.handleComponentEvent(ActivationSuccessful{component_index}); + const auto component = graph_.getProcessInfoNode(component_index); + EXPECT_CALL(process_interface_, requestTermination) + .WillOnce(DoAll( + InvokeWithoutArgs([component] { + static_cast(component->tryHandleTermination(134)); + }), + Return(osal::OsalReturnType::kSuccess))); + // The active component then crashes graph_.handleComponentEvent(UnexpectedTermination{component_index}); diff --git a/tests/integration/crash_ignores_dependents/BUILD b/tests/integration/crash_ignores_dependents/BUILD new file mode 100644 index 0000000000..039993802c --- /dev/null +++ b/tests/integration/crash_ignores_dependents/BUILD @@ -0,0 +1,47 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_binary( + name = "test_process", + srcs = ["test_process.cpp"], + deps = [ + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +cc_binary( + name = "process_crashing_once", + srcs = ["process_crashing_once.cpp"], + deps = [ + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "crash_ignores_dependents", + srcs = ["crash_ignores_dependents.py"], + binaries = [ + ":config", + ":test_process", + ":process_crashing_once", + "//score/launch_manager", + ], + config = ":crash_ignores_dependents.json", +) diff --git a/tests/integration/crash_ignores_dependents/crash_ignores_dependents.json b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.json new file mode 100644 index 0000000000..1cbf6f019c --- /dev/null +++ b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.json @@ -0,0 +1,96 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/crash_ignores_dependents", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "test_process": { + "component_properties": { + "binary_name": "test_process", + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": true, + "alive_supervision": { + "min_indications": 0 + } + }, + "depends_on": ["component_crashing_once"] + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "test_process" + } + } + }, + "component_crashing_once": { + "component_properties": { + "binary_name": "process_crashing_once", + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": true + } + }, + "deployment_config": { + "environmental_variables": { + "PROCESSIDENTIFIER": "component_crashing_once" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "test_process" + ] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [ + "test_process" + ] + } +} diff --git a/tests/integration/crash_ignores_dependents/crash_ignores_dependents.py b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.py new file mode 100644 index 0000000000..0c55c3d56c --- /dev/null +++ b/tests/integration/crash_ignores_dependents/crash_ignores_dependents.py @@ -0,0 +1,45 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + test_type="interface-test", + derivation_technique="design-analysis", +) +def test_crash_ignores_dependents( + target, setup_test, assert_test_results, remote_test_dir +): + """ + Objective: Verifies that the launch manager does not restart a process if a process it depends on crashes. + + A process crashes after run target activation completes and proceeds normally the second time it is launched. + Expected Behaviour: The process that depends on it is not interrupted or restarted. + """ + + new_config_path = str(remote_test_dir / "etc/crash_ignores_dependents.bin") + + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=remote_test_dir.parent / "test_end", + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=6.0, + ) + + assert_test_results({"test_process.xml", "process_crashing_once.xml"}) diff --git a/tests/integration/crash_ignores_dependents/process_crashing_once.cpp b/tests/integration/crash_ignores_dependents/process_crashing_once.cpp new file mode 100644 index 0000000000..e3ff6f1826 --- /dev/null +++ b/tests/integration/crash_ignores_dependents/process_crashing_once.cpp @@ -0,0 +1,50 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include + +TEST(CrashIgnoresDependents, CrashingProcess) +{ + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + // Limitation: we can't wait for run target activation to complete + sleep(1); + + TEST_STEP("Crash if we haven't crashed yet") + { + const std::string_view crash_file = "process_crashed"; + + if (!std::filesystem::exists(crash_file)) + { + std::cout << "Process crashing..." << std::endl; + if (!touch_file(crash_file)) + { + std::cout << "Failed to deploy marker file!" << std::endl; + } + exit(1); + } + + ASSERT_TRUE(touch_file("process_started_normally")); + std::cout << "Process finishing normally" << std::endl; + } +} + +int main() +{ + TestRunner(__FILE__, TerminationBehavior::kContinue).RunTests(); +} diff --git a/tests/integration/crash_ignores_dependents/test_process.cpp b/tests/integration/crash_ignores_dependents/test_process.cpp new file mode 100644 index 0000000000..57a276c13a --- /dev/null +++ b/tests/integration/crash_ignores_dependents/test_process.cpp @@ -0,0 +1,57 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include +#include +#include + +TEST(CrashIgnoresDependents, TestProcess) +{ + const std::string_view started_file = "test_process_started"; + const std::string_view crash_file = "process_crashed"; + + TEST_STEP("Check this is the first start") + { + ASSERT_TRUE(check_clean({test_end_location, started_file, crash_file})) + << "Process was started more than once!"; + + ASSERT_TRUE(touch_file(started_file)) << "Failed to deploy file!"; + } + + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + // Wait for the crashing process to deploy its first file + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!std::filesystem::exists(crash_file) && std::chrono::steady_clock::now() < deadline) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } + // Then + TEST_STEP("Verify process crashed while this was running") + { + EXPECT_TRUE(std::filesystem::exists(crash_file)) << "Process did not actually crash"; + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kContinue, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/fallback_to_same_target_restarts/BUILD b/tests/integration/fallback_to_same_target_restarts/BUILD new file mode 100644 index 0000000000..e2faa68258 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/BUILD @@ -0,0 +1,47 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") +load("//tests/utils/bazel:integration.bzl", "integration_test") + +cc_binary( + name = "control_client_mock", + srcs = ["control_client_mock.cpp"], + deps = [ + "//score/launch_manager:control_cc", + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +cc_binary( + name = "process_crashing_once", + srcs = ["process_crashing_once.cpp"], + deps = [ + "//score/launch_manager:lifecycle_cc", + "//tests/utils/test_helper", + "@googletest//:gtest_main", + ], +) + +integration_test( + name = "fallback_to_same_target_restarts", + srcs = ["fallback_to_same_target_restarts.py"], + binaries = [ + ":config", + ":control_client_mock", + ":process_crashing_once", + "//score/launch_manager", + ], + config = ":fallback_to_same_target_restarts.json", +) diff --git a/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp b/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp new file mode 100644 index 0000000000..a00645d617 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/control_client_mock.cpp @@ -0,0 +1,69 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include +#include +#include +#include +#include + +// Given a correct configuration with: +// - An initial Run Target named "Startup" containing "control_client_mock" +// - A Run Target named "run_target_crashing_app_on_runtime" containing "control_client_mock" and +// "component_crashing_once" + +TEST(FallbackToSameTargetRestarts, ControlClientMock) +{ + score::mw::lifecycle::ControlClient client; + + const std::string_view process_file = "process_started_normally"; + + ASSERT_TRUE(check_clean({test_end_location, process_file})); + // Establish communication with launch manager + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + TEST_STEP("Start crashing process") + { + score::cpp::stop_token stop_token; + auto result = client.ActivateRunTarget("run_target_crashing_app_on_runtime").Get(stop_token); + EXPECT_TRUE(result.has_value()) << "Activating target run_target_crashing_app_on_runtime failed: " + << result.error().Message(); + } + // When the process crashes, wait for the fallback to be activated. + { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (!std::filesystem::exists(process_file) && std::chrono::steady_clock::now() < deadline) + { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + } + } + // Then + TEST_STEP("Verify process was restarted") + { + EXPECT_TRUE(std::filesystem::exists(process_file)) << "Process did not restart successfully"; + } + TEST_STEP("Activate RunTarget Off") + { + client.ActivateRunTarget("Off"); + } +} + +int main() +{ + return TestRunner(__FILE__, TerminationBehavior::kWait, TerminationNotification::kTestEnd).RunTests(); +} diff --git a/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.json b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.json new file mode 100644 index 0000000000..173ee643c8 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.json @@ -0,0 +1,109 @@ +{ + "schema_version": 1, + "defaults": { + "deployment_config": { + "bin_dir": "/tmp/tests/fallback_to_same_target_restarts", + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "ready_recovery_action": { + "restart": { + "number_of_attempts": 0 + } + }, + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + }, + "environmental_variables": { + "LD_LIBRARY_PATH": "/opt/lib" + }, + "sandbox": { + "uid": 0, + "gid": 0, + "scheduling_policy": "SCHED_OTHER", + "scheduling_priority": 0 + } + }, + "component_properties": { + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": false, + "alive_supervision": { + "reporting_cycle": 0.1, + "min_indications": 1, + "max_indications": 3, + "failed_cycles_tolerance": 1 + } + }, + "ready_condition": { + "process_state": "Running" + } + } + }, + "components": { + "control_client_mock": { + "component_properties": { + "binary_name": "control_client_mock", + "application_profile": { + "application_type": "State_Manager", + "alive_supervision": { + "min_indications": 0 + } + } + }, + "deployment_config": { + "ready_timeout": 1.0, + "shutdown_timeout": 1.0, + "environmental_variables": { + "PROCESSIDENTIFIER": "control_client_mock" + } + } + }, + "component_crashing_once": { + "component_properties": { + "binary_name": "process_crashing_once", + "application_profile": { + "application_type": "Reporting", + "is_self_terminating": true + } + }, + "deployment_config": { + "environmental_variables": { + "PROCESSIDENTIFIER": "component_crashing_once" + } + } + } + }, + "run_targets": { + "Startup": { + "depends_on": [ + "control_client_mock" + ] + }, + "run_target_crashing_app_on_runtime": { + "depends_on": [ + "control_client_mock", + "component_crashing_once" + ], + "recovery_action": { + "switch_run_target": { + "run_target": "fallback_run_target" + } + } + }, + "Off": { + "depends_on": [] + } + }, + "initial_run_target": "Startup", + "alive_supervision": { + "evaluation_cycle": 0.05 + }, + "fallback_run_target": { + "depends_on": [ + "control_client_mock", + "component_crashing_once" + ] + } +} diff --git a/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py new file mode 100644 index 0000000000..c2716cd011 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/fallback_to_same_target_restarts.py @@ -0,0 +1,45 @@ +# ******************************************************************************* +# Copyright (c) 2026 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) distributed with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made available under the +# terms of the Apache License Version 2.0 which is available at +# https://www.apache.org/licenses/LICENSE-2.0 +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************* +from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed +from tests.utils.testing_utils.setup_test import setup_test +from tests.utils.testing_utils.test_results import assert_test_results +from attribute_plugin import add_test_properties + + +@add_test_properties( + fully_verifies=[], + test_type="interface-test", + derivation_technique="error-guessing", +) +def test_fallback_to_same_target_restarts( + target, setup_test, assert_test_results, remote_test_dir +): + """ + Objective: Verifies that the launch manager correctly restarts a crashed process if it is active in the fallback state. + + A process crashes after run target activation completes and proceeds normally the second time it is launched. + Expected Behaviour: The process is relaunched and completes normally. + """ + + new_config_path = str(remote_test_dir / "etc/fallback_to_same_target_restarts.bin") + + run_until_file_deployed( + target=target, + binary_path=str(remote_test_dir / "launch_manager"), + file_path=remote_test_dir.parent / "test_end", + cwd=str(remote_test_dir), + args=["-c", new_config_path], + timeout_s=4.0, + ) + + assert_test_results({"control_client_mock.xml", "process_crashing_once.xml"}) diff --git a/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp b/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp new file mode 100644 index 0000000000..d9b8e14245 --- /dev/null +++ b/tests/integration/fallback_to_same_target_restarts/process_crashing_once.cpp @@ -0,0 +1,50 @@ +/******************************************************************************** + * Copyright (c) 2026 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +#include + +#include "tests/utils/test_helper/test_helper.hpp" +#include + +TEST(FallbackToSameTargetRestarts, CrashingProcess) +{ + TEST_STEP("Report running") + { + score::mw::lifecycle::report_running(); + } + + // Limitation: we can't wait for run target activation to complete + sleep(1); + + TEST_STEP("Crash if we haven't crashed yet") + { + const std::string_view crash_file = "process_crashed"; + + if (!std::filesystem::exists(crash_file)) + { + std::cout << "Process crashing..." << std::endl; + if (!touch_file(crash_file)) + { + std::cout << "Failed to deploy marker file!" << std::endl; + } + exit(1); + } + + ASSERT_TRUE(touch_file("process_started_normally")); + std::cout << "Process finishing normally" << std::endl; + } +} + +int main() +{ + TestRunner(__FILE__, TerminationBehavior::kContinue).RunTests(); +} diff --git a/tests/utils/test_helper/test_helper.hpp b/tests/utils/test_helper/test_helper.hpp index b0ba2a732c..ff7c397478 100644 --- a/tests/utils/test_helper/test_helper.hpp +++ b/tests/utils/test_helper/test_helper.hpp @@ -19,7 +19,7 @@ #include /// @return File path to an xml adjacent to the input file path -std::string xmlPath(const std::string_view file) +inline std::string xmlPath(const std::string_view file) { return std::filesystem::path{file}.filename().stem().string() + ".xml"; } diff --git a/tests/utils/testing_utils/run_until_file_deployed.py b/tests/utils/testing_utils/run_until_file_deployed.py index a5ba530a88..d765e11038 100644 --- a/tests/utils/testing_utils/run_until_file_deployed.py +++ b/tests/utils/testing_utils/run_until_file_deployed.py @@ -25,7 +25,6 @@ def run_until_file_deployed( file_path: str, timeout_s: float = 30.0, poll_interval_s: float = 0.5, - stop_timeout_s: float = 3.0, args=None, cwd: str = "/", ) -> AsyncProcess: @@ -36,8 +35,6 @@ def run_until_file_deployed( :param file_path: path of the file to wait for on the target. :param timeout_s: maximum seconds to wait for the file (default: 30). :param poll_interval_s: seconds between file checks (default: 0.5). - :param stop_timeout_s: maximum seconds to wait for the process to terminate - after SIGTERM (default: 3). :param args: optional list of arguments to pass to the binary. :param cwd: working directory on the target (default: "/"). :return: the stopped :class:`AsyncProcess` handle. @@ -62,21 +59,7 @@ def run_until_file_deployed( exit_code, _ = target.execute(f"test -f {file_path}") if exit_code == 0: - # Kill the entire process group so that children (e.g. the actual - # daemon binary launched under fakeroot) receive SIGTERM and can - # run their cleanup code before exiting. - kill_cmd = f"kill -TERM {proc.pid()}" - res, _ = target.execute(kill_cmd) - assert res == 0, "Couldn't kill lcm with SIGTERM" - try: - # wait() raises RuntimeError if the process is still running after - # stop_timeout_s. - exit_code = proc.wait(timeout_s=stop_timeout_s) - except RuntimeError as exc: - proc.stop() # escalate to SIGKILL so we don't leak the process - assert False, ( - f"Process '{binary_path}' still running {stop_timeout_s}s after SIGTERM: {exc}" - ) + exit_code = proc.stop() assert exit_code == 0, ( f"LCM did not exit cleanly, it died with code {exit_code}" )