Extend crash_on_startup test to also cover comp_req__launch_man__retries_configurable

tests/integration/crash_on_startup/BUILD

@@ -24,29 +24,13 @@ cc_binary(
     ],
 )
 
-cc_binary(
-    name = "process_crashing_on_startup_twice",
-    srcs = ["process_crashing_on_startup_twice.cpp"],
-    deps = [
-        "//score/launch_manager:lifecycle_cc",
-        "//tests/utils/test_helper",
-        "@googletest//:gtest_main",
-    ],
-)
-
-cc_binary(
-    name = "process_crashing_on_startup_always",
-    srcs = ["process_crashing_on_startup_always.cpp"],
-)
-
 integration_test(
     name = "crash_on_startup",
     srcs = ["crash_on_startup.py"],
     binaries = [
         ":control_client_mock",
-        ":process_crashing_on_startup_always",
-        ":process_crashing_on_startup_twice",
         "//score/launch_manager",
+        "//tests/utils/test_helper:process_crashing_on_startup_n_times",
         "//tests/utils/test_helper:verification_process",
     ],
     config = ":crash_on_startup.json",

tests/integration/crash_on_startup/control_client_mock.cpp

@@ -21,34 +21,42 @@ TEST(CrashOnStartup, ControlClientMock)
 {
     score::mw::lifecycle::ControlClient client;
 
-    ASSERT_TRUE(check_clean({crashed_once_file, crashed_twice_file, test_end_location, fallback_file}));
+    ASSERT_TRUE(
+        check_clean({crashCountPath(1), crashCountPath(2), crashCountPath(3), test_end_location, fallback_file}));
 
     TEST_STEP("Report running")
     {
         score::mw::lifecycle::report_running();
     }
 
-    // Given a process that crashes on startup twice
-    TEST_STEP("Launch process crashing on startup twice")
+    // Given a process that crashes on startup n times, but is configured to retry n times - so it eventually
+    // succeeds. The behaviour is identical for the different crash counts, so it is parameterized over the
+    // corresponding run targets. Each run target's process persists its crash count in its own file, so the
+    // run targets do not interfere with each other.
+    for (const std::string_view run_target :
+         {"run_target_crash_on_startup_two_times", "run_target_crash_on_startup_three_times"})
     {
-        score::cpp::stop_token stop_token;
-        auto result = client.ActivateRunTarget("run_target_crash_on_startup_twice").Get(stop_token);
-        // Then, the LM should restart it and eventually succeed
-        EXPECT_TRUE(result.has_value()) << "Activating run_target_crash_on_startup_twice failed: "
-                                        << result.error().Message();
-    }
+        TEST_STEP(std::string{"Launch "} + std::string{run_target})
+        {
+            score::cpp::stop_token stop_token;
+            auto result = client.ActivateRunTarget(run_target).Get(stop_token);
+            // Then, the LM should restart it and eventually succeed
+            EXPECT_TRUE(result.has_value()) << "Activating " << run_target << " failed: " << result.error().Message();
+        }
 
-    TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully")
-    {
-        EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet";
+        TEST_STEP("Verify fallback run target was not activated, i.e. process eventually started successfully")
+        {
+            EXPECT_FALSE(std::filesystem::exists(fallback_file)) << "Fallback run target should not be activated yet";
+        }
     }
 
-    // Given a process that crashes on startup more times than the configured restart attempts
-    TEST_STEP("Attempt to launch process crashing on startup always")
+    // Given a process that crashes on startup but is not allowed to retry (number_of_attempts=0)
+    TEST_STEP("Attempt to launch process crashing on startup without retries")
     {
         score::cpp::stop_token stop_token;
-        auto result = client.ActivateRunTarget("run_target_crash_on_startup_always").Get(stop_token);
-        EXPECT_FALSE(result.has_value()) << "Expected run_target_crash_on_startup_always activation to fail";
+        auto result = client.ActivateRunTarget("run_target_crash_on_startup_once_but_no_retries").Get(stop_token);
+        EXPECT_FALSE(result.has_value())
+            << "Expected run_target_crash_on_startup_once_but_no_retries activation to fail";
     }
     // Limitation: we cannot wait for the transition to fallback to complete
     sleep(1);

tests/integration/crash_on_startup/crash_on_startup.json

@@ -3,8 +3,6 @@
     "defaults": {
         "deployment_config": {
             "bin_dir": "/tmp/tests/crash_on_startup",
-            "ready_timeout": 1.0,
-            "shutdown_timeout": 1.0,
             "ready_recovery_action": {
                 "restart": {
                     "number_of_attempts": 0
@@ -24,21 +22,6 @@
                 "scheduling_policy": "SCHED_OTHER",
                 "scheduling_priority": 0
             }
-        },
-        "component_properties": {
-            "application_profile": {
-                "application_type": "Reporting",
-                "is_self_terminating": false,
-                "alive_supervision": {
-                    "reporting_cycle": 0.1,
-                    "min_indications": 1,
-                    "max_indications": 3,
-                    "failed_cycles_tolerance": 1
-                }
-            },
-            "ready_condition": {
-                "process_state": "Running"
-            }
         }
     },
     "components": {
@@ -62,40 +45,51 @@
         },
         "process_crashing_on_startup_twice": {
             "component_properties": {
-                "binary_name": "process_crashing_on_startup_twice",
+                "binary_name": "process_crashing_on_startup_n_times",
                 "application_profile": {
                     "application_type": "Reporting"
-                }
+                },
+                "process_arguments": [
+                    "2"
+                ]
             },
             "deployment_config": {
                 "ready_recovery_action": {
                     "restart": {
                         "number_of_attempts": 2
                     }
-                },
-                "environmental_variables": {
-                    "PROCESSIDENTIFIER": "process_crashing_on_startup_twice"
                 }
             }
         },
-        "process_crashing_on_startup_always": {
+        "process_crashing_on_startup_three_times": {
             "component_properties": {
-                "binary_name": "process_crashing_on_startup_always",
+                "binary_name": "process_crashing_on_startup_n_times",
                 "application_profile": {
                     "application_type": "Reporting"
-                }
+                },
+                "process_arguments": [
+                    "3"
+                ]
             },
             "deployment_config": {
                 "ready_recovery_action": {
                     "restart": {
-                        "number_of_attempts": 2
+                        "number_of_attempts": 3
                     }
-                },
-                "environmental_variables": {
-                    "PROCESSIDENTIFIER": "process_crashing_on_startup_always"
                 }
             }
         },
+        "process_crashing_on_startup_no_retries": {
+            "component_properties": {
+                "binary_name": "process_crashing_on_startup_n_times",
+                "application_profile": {
+                    "application_type": "Reporting"
+                },
+                "process_arguments": [
+                    "1"
+                ]
+            }
+        },
         "verification_component": {
             "component_properties": {
                 "binary_name": "verification_process",
@@ -118,16 +112,22 @@
                 "control_client_mock"
             ]
         },
-        "run_target_crash_on_startup_twice": {
+        "run_target_crash_on_startup_two_times": {
             "depends_on": [
                 "control_client_mock",
                 "process_crashing_on_startup_twice"
             ]
         },
-        "run_target_crash_on_startup_always": {
+        "run_target_crash_on_startup_three_times": {
+            "depends_on": [
+                "control_client_mock",
+                "process_crashing_on_startup_three_times"
+            ]
+        },
+        "run_target_crash_on_startup_once_but_no_retries": {
             "depends_on": [
                 "control_client_mock",
-                "process_crashing_on_startup_always"
+                "process_crashing_on_startup_no_retries"
             ],
             "recovery_action": {
                 "switch_run_target": {

tests/integration/crash_on_startup/crash_on_startup.py

@@ -12,28 +12,36 @@
 # *******************************************************************************
 from tests.utils.testing_utils.run_until_file_deployed import run_until_file_deployed
 from tests.utils.testing_utils.setup_test import setup_test
-from tests.utils.testing_utils.test_results import assert_test_results
+from tests.utils.testing_utils.test_results import (
+    assert_test_results,
+    get_testcase_property,
+)
 from attribute_plugin import add_test_properties
 
 
 @add_test_properties(
     fully_verifies=[
         "comp_req__launch_man__failure_detect",
+        "comp_req__launch_man__retries_configurable",
         "feat_req__lifecycle__recov_run_target_switch",
     ],
     partially_verifies=["feat_req__lifecycle__recovery_action_support"],
     test_type="requirements-based",
     derivation_technique="requirements-analysis",
 )
-def test_crash_on_startup(target, setup_test, assert_test_results, remote_test_dir):
+def test_crash_on_startup(
+    target, setup_test, assert_test_results, remote_test_dir, test_output_dir
+):
     """
     Objective: Verifies that the launch manager correctly handles processes that crash before reporting running.
 
     Case 1: Process crashes before Running state but eventually starts up successfully before the configured number of restart attempts is exceeded.
-    Expected Behaviour: Process startup successful, RunTarget activation successful
+    This is verified with two different components: One with the process crashing twice and the other three times before successfully starting up.
+    The number of restart attempts is configured to be 2 and 3 respectively for these two components.
+    Expected Behaviour: Process startup successful, run target activation successful
 
-    Case 2: Process keeps crashing, exceeding the number of restart attempts.
-    Expected Behaviour: Process startup fails, LaunchManager executes recovery action.
+    Case 2: Component has no restart attempts configured, but crashes once.
+    Expected Behaviour: Process startup fails and therefore run target activation fails. Launch manager executes recovery action which switches to fallback run target.
     """
 
     config_path = str(remote_test_dir / "etc/crash_on_startup.bin")
@@ -47,6 +55,19 @@ def test_crash_on_startup(target, setup_test, assert_test_results, remote_test_d
         timeout_s=10.0,
     )
 
-    assert_test_results(
-        {"control_client_mock.xml", "process_crashing_on_startup_twice.xml"}
-    )
+    # Each crashing process writes its own report file named after the number of times it crashes, so the
+    # reports of the different run targets no longer overwrite each other. The process crashing once is not
+    # allowed to retry, but still writes its report before crashing.
+    crash_report_names = {
+        n: f"process_crashing_on_startup_n_times_n_equals_{n}.xml" for n in (1, 2, 3)
+    }
+    assert_test_results({"control_client_mock.xml", *crash_report_names.values()})
+
+    # The number of crashes is recorded in each report and must match the configured crash count.
+    for n, report_name in crash_report_names.items():
+        crash_count = get_testcase_property(
+            test_output_dir / report_name, "crash_count"
+        )
+        assert crash_count == str(n), (
+            f"Expected {n} crashes in {report_name}, got {crash_count}"
+        )

tests/utils/test_helper/BUILD

@@ -63,3 +63,14 @@ cc_binary(
         ":test_helper",
     ],
 )
+
+cc_binary(
+    name = "process_crashing_on_startup_n_times",
+    srcs = ["process_crashing_on_startup_n_times.cpp"],
+    visibility = ["//tests:__subpackages__"],
+    deps = [
+        ":test_helper",
+        "//score/launch_manager:lifecycle_cc",
+        "@googletest//:gtest_main",
+    ],
+)