[DRAFT] Run conformance tests as one whole-suite invocation per module

codeplain_REST_api.py

@@ -335,6 +335,7 @@ def render_conformance_tests(
             "conformance_tests_json": conformance_tests_json,
             "all_acceptance_tests": all_acceptance_tests,
             "existing_conformance_tests_files": existing_conformance_tests_files,
+            "conformance_tests_paths_relative_to_module_root": True,
         }
 
         response = self.post_request(endpoint_url, headers, payload, run_state)
@@ -510,26 +511,3 @@ def fail_functional_requirement(self, frid, module_name: str, run_state: RunStat
         }
 
         return self.post_request(endpoint_url, headers, payload, run_state)
-
-    def summarize_finished_conformance_tests(
-        self,
-        frid,
-        plain_source_tree,
-        linked_resources,
-        conformance_test_files_content,
-        module_name: str,
-        required_modules,
-        run_state: RunState,
-    ):
-        endpoint_url = f"{self.api_url}/summarize_finished_conformance_tests"
-        headers = {"X-API-Key": self.api_key, "Content-Type": "application/json"}
-        payload = {
-            "frid": frid,
-            "plain_source_tree": plain_source_tree,
-            "linked_resources": linked_resources,
-            "conformance_test_files_content": conformance_test_files_content,
-            "module_name": module_name,
-            "required_modules": required_modules,
-        }
-
-        return self.post_request(endpoint_url, headers, payload, run_state)

render_machine/actions/render_conformance_tests.py

@@ -8,11 +8,12 @@
 from render_machine.actions.base_action import BaseAction
 from render_machine.implementation_code_helpers import ImplementationCodeHelpers
 from render_machine.render_context import RenderContext
-from render_machine.render_types import AcceptanceTestPhase, TestExecutionPhase
+from render_machine.render_types import AcceptanceTestPhase, RenderError, TestExecutionPhase
 
 
 class RenderConformanceTests(BaseAction):
     SUCCESSFUL_OUTCOME = "conformance_test_rendered"
+    RESPONSE_VALIDATION_FAILED_OUTCOME = "conformance_test_response_validation_failed"
 
     def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
         if self._should_render_conformance_tests(render_context):
@@ -123,33 +124,64 @@ def _render_conformance_tests(self, render_context: RenderContext):
             if not file_name.startswith(current_subfolder_prefix)
         }
 
-        response_files, implementation_plan_summary = render_context.codeplain_api.render_conformance_tests(
-            render_context.frid_context.frid,
-            render_context.conformance_tests_running_context.current_testing_frid,
-            render_context.plain_source_tree,
-            render_context.frid_context.linked_resources,
-            existing_files_content,
-            memory_files_content,
-            render_context.module_name,
-            render_context.get_required_modules_functionalities(),
-            conformance_tests_folder_name,
-            render_context.conformance_tests_running_context.get_conformance_tests_json(
-                render_context.conformance_tests_running_context.current_testing_module_name
-            ),
-            all_acceptance_tests,
-            existing_conformance_tests_files,
-            run_state=render_context.run_state,
+        current_subfolder_name = os.path.basename(conformance_tests_folder_name)
+        module_conformance_tests_folder = render_context.conformance_tests.get_module_conformance_tests_folder(
+            render_context.module_name
         )
 
+        for attempt in range(2):
+            response_files, implementation_plan_summary = render_context.codeplain_api.render_conformance_tests(
+                render_context.frid_context.frid,
+                render_context.conformance_tests_running_context.current_testing_frid,
+                render_context.plain_source_tree,
+                render_context.frid_context.linked_resources,
+                existing_files_content,
+                memory_files_content,
+                render_context.module_name,
+                render_context.get_required_modules_functionalities(),
+                conformance_tests_folder_name,
+                render_context.conformance_tests_running_context.get_conformance_tests_json(
+                    render_context.conformance_tests_running_context.current_testing_module_name
+                ),
+                all_acceptance_tests,
+                existing_conformance_tests_files,
+                run_state=render_context.run_state,
+            )
+
+            violations = render_context.conformance_tests.find_response_file_violations(
+                render_context.module_name,
+                current_subfolder_name,
+                response_files,
+            )
+
+            if not violations:
+                break
+
+            console.warning(
+                "Generated conformance test files violate the suite layout rules:\n  "
+                + "\n  ".join(violations)
+                + ("\nRetrying the generation." if attempt == 0 else "")
+            )
+        else:
+            return (
+                self.RESPONSE_VALIDATION_FAILED_OUTCOME,
+                RenderError.encode(
+                    message="Generated conformance test files repeatedly violated the suite layout rules "
+                    "(files outside the functionality's subfolder or invalid changes to shared setup files).",
+                    error_type="CONFORMANCE_TESTS_VALIDATION_ERROR",
+                    violations="\n".join(violations),
+                ).to_payload(),
+            )
+
         render_context.conformance_tests_running_context.current_testing_frid_high_level_implementation_plan = (
             implementation_plan_summary
         )
 
-        file_utils.store_response_files(conformance_tests_folder_name, response_files, [])
+        file_utils.store_response_files(module_conformance_tests_folder, response_files, [])
 
         console.print_files(
             "Conformance test files generated:",
-            conformance_tests_folder_name,
+            module_conformance_tests_folder,
             response_files,
             style=console.OUTPUT_STYLE,
         )

render_machine/actions/run_conformance_tests.py

@@ -4,11 +4,20 @@
 import render_machine.render_utils as render_utils
 from plain2code_console import console
 from render_machine.actions.base_action import BaseAction
+from render_machine.failure_attribution import detect_layout_failure
 from render_machine.render_context import RenderContext
 from render_machine.render_types import RenderError
 
 UNRECOVERABLE_ERROR_EXIT_CODES = [69]
 
+LAYOUT_FAILURE_MESSAGE = (
+    "Conformance test suites of this module could not be discovered or run together. "
+    "This usually means the conformance tests were generated by an older version of the renderer - "
+    "delete the module's conformance tests folder so they get regenerated on the next render. "
+    "If the project uses a custom conformance tests script, make sure it runs all tests found "
+    "under the folder it receives as its second argument, including tests in subfolders."
+)
+
 
 class RunConformanceTests(BaseAction):
 
@@ -19,26 +28,16 @@ class RunConformanceTests(BaseAction):
     def execute(self, render_context: RenderContext, _previous_action_payload: Any | None):
         conformance_tests_script = os.path.normpath(render_context.conformance_tests_script)
 
-        if render_context.module_name == render_context.conformance_tests_running_context.current_testing_module_name:
-            conformance_tests_folder_name = (
-                render_context.conformance_tests_running_context.get_current_conformance_test_folder_name()
-            )
-        else:
-            [conformance_tests_folder_name, _] = (
-                render_context.conformance_tests.get_source_conformance_test_folder_name(
-                    render_context.module_name,
-                    render_context.required_modules,
-                    render_context.conformance_tests_running_context.current_testing_module_name,
-                    render_context.conformance_tests_running_context.get_current_conformance_test_folder_name(),
-                )
-            )
+        conformance_tests_folder_name = render_context.conformance_tests.get_module_suite_run_folder(
+            render_context.module_name,
+            render_context.required_modules,
+            render_context.conformance_tests_running_context.current_testing_module_name,
+        )
 
         console.info(
             f"Running conformance tests script {conformance_tests_script} "
-            + f"for {conformance_tests_folder_name} ("
-            + f"functionality {render_context.conformance_tests_running_context.current_testing_frid} "
-            + f"in module {render_context.conformance_tests_running_context.current_testing_module_name}"
-            + ")."
+            + f"for the test suite {conformance_tests_folder_name} "
+            + f"of module {render_context.conformance_tests_running_context.current_testing_module_name}."
         )
         exit_code, conformance_tests_issue, conformance_tests_temp_log_file_path = render_utils.execute_script(
             conformance_tests_script,
@@ -54,21 +53,23 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any |
         )
         render_context.script_execution_history.should_update_script_outputs = True
 
-        render_context.memory_manager.create_conformance_tests_memory(
-            render_context, exit_code, conformance_tests_issue
-        )
-
         if exit_code == 0:
+            render_context.memory_manager.create_conformance_tests_memory(
+                render_context, exit_code, conformance_tests_issue
+            )
+            # A passing whole-suite run of the module being rendered covers the FRID being
+            # implemented, so its unresolved memory entries can be cleared.
             if (
                 render_context.conformance_tests_running_context.current_testing_module_name
                 == render_context.module_name
-                and render_context.conformance_tests_running_context.current_testing_frid
-                == render_context.frid_context.frid
             ):
                 render_context.memory_manager.delete_unresolved_memory_files()
             return self.SUCCESSFUL_OUTCOME, None
 
         if exit_code in UNRECOVERABLE_ERROR_EXIT_CODES:
+            render_context.memory_manager.create_conformance_tests_memory(
+                render_context, exit_code, conformance_tests_issue
+            )
             console.error(conformance_tests_issue)
             return (
                 self.UNRECOVERABLE_ERROR_OUTCOME,
@@ -80,4 +81,23 @@ def execute(self, render_context: RenderContext, _previous_action_payload: Any |
                 ).to_payload(),
             )
 
-        return self.FAILED_OUTCOME, {"previous_conformance_tests_issue": conformance_tests_issue}
+        if detect_layout_failure(conformance_tests_issue):
+            console.error(conformance_tests_issue)
+            return (
+                self.UNRECOVERABLE_ERROR_OUTCOME,
+                RenderError.encode(
+                    message=LAYOUT_FAILURE_MESSAGE,
+                    error_type="ENVIRONMENT_ERROR",
+                    script=conformance_tests_script,
+                    issue=conformance_tests_issue,
+                ).to_payload(),
+            )
+
+        # Attribute the failure to a FRID (re-pointing the running context for the fix loop)
+        # before creating memory, so the memory entry is keyed to the implicated FRID.
+        conformance_tests_evidence = render_context.route_conformance_failure_to_frid(conformance_tests_issue)
+        render_context.memory_manager.create_conformance_tests_memory(
+            render_context, exit_code, conformance_tests_issue
+        )
+
+        return self.FAILED_OUTCOME, {"previous_conformance_tests_issue": conformance_tests_evidence}