run_time_bootstrap.py

"""Bootstrap the ENT RAG run-time implementation.

Run from the project root:
    python run_time_bootstrap.py

This script creates:
    run_time/scripts/                         Python run-time implementation files
    run_time/scripts/evaluation_scripts/      Python evaluation implementation files
    evaluation/                               Persistent evaluation outputs

The run_time directory is intentionally volatile and may be reset on bootstrap
re-runs. The evaluation directory is intentionally persistent and is never
removed by this bootstrap.
"""

from __future__ import annotations

from pathlib import Path
from textwrap import dedent

import shutil
import subprocess
import sys


PROJECT_ROOT = Path(__file__).resolve().parent

# 1. Volatile run-time implementation path
RUN_TIME_DIR = PROJECT_ROOT / "run_time"
RUN_TIME_SCRIPT_DIR = RUN_TIME_DIR / "scripts"
RUN_TIME_EVALUATION_SCRIPT_DIR = RUN_TIME_SCRIPT_DIR / "evaluation_scripts"

# 2. Persistent evaluation output path
EVALUATION_DIR = PROJECT_ROOT / "evaluation"
EVALUATION_CHECKPOINTS_DIR = EVALUATION_DIR / "checkpoints"
EVALUATION_REPORTS_DIR = EVALUATION_DIR / "reports"
EVALUATION_FIGURES_DIR = EVALUATION_REPORTS_DIR / "figures"
EVALUATION_TABLES_DIR = EVALUATION_REPORTS_DIR / "tables"
SECRETS_PATH = PROJECT_ROOT / ".secrets"

REQUIRED_SECRET_KEYS = [
    "HF_TOKEN",
    "OPENAI_API_KEY",
]

# Do you want to reset only the volatile run_time directory every run?
RESET_RUN_TIME_DIR = True
# Do you want to overwrite the existing runtime modules every run?
OVERWRITE_MODULES = True
# Do you want to run the evaluation pipeline immediately after module creation?
RUN_RUNTIME_PIPELINE_AFTER_BOOTSTRAP = False

RUN_TIME_SCRIPT_EXEC_PY_PATH = RUN_TIME_SCRIPT_DIR / "runtime_pipeline.py"

DIRECTORIES = [
    RUN_TIME_DIR,
    RUN_TIME_SCRIPT_DIR,
    RUN_TIME_EVALUATION_SCRIPT_DIR,
    EVALUATION_DIR,
    EVALUATION_CHECKPOINTS_DIR,
    EVALUATION_REPORTS_DIR,
    EVALUATION_FIGURES_DIR,
    EVALUATION_TABLES_DIR,
]

FILES = {}
FILES[RUN_TIME_SCRIPT_DIR / "__init__.py"] = ""
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / "__init__.py"] = ""

# -----------------------------------------------------------------------------
# config.py
FILES[RUN_TIME_SCRIPT_DIR / 'config.py'] = dedent(r'''
"""Configuration values for the run-time ENT RAG chatbot and evaluation pipeline.

This module is intentionally written first by ``run_time_bootstrap.py`` and is
imported by every other run-time module. Keep path, model, generation, and
report-output declarations here rather than duplicating them across scripts.
"""

from __future__ import annotations

import os
from pathlib import Path

from dotenv import load_dotenv


PROJECT_ROOT = Path(__file__).resolve().parents[2]

# -----------------------------------------------------------------------------
# 1. Dataset and secrets
DATASET_PATH = PROJECT_ROOT / "dataset" / "2P_ENT_QuAD.csv"
SECRETS_PATH = PROJECT_ROOT / ".secrets"

# -----------------------------------------------------------------------------
# 2. Build-time RAG digest artifacts
BUILD_TIME_DIR = PROJECT_ROOT / "build_time"
RAG_DIGEST_DIR = BUILD_TIME_DIR / "rag_digest"

RAW_IMAGES_DIR = RAG_DIGEST_DIR / "raw_images"
LOOKUP_TABLES_DIR = RAG_DIGEST_DIR / "lookup_tables"
METADATA_DIR = RAG_DIGEST_DIR / "metadata"
FAISS_INDEXES_DIR = RAG_DIGEST_DIR / "faiss_indexes"
EMBEDDING_MATRICES_DIR = RAG_DIGEST_DIR / "embedding_matrices"
BUILD_TIME_REPORTS_DIR = RAG_DIGEST_DIR / "reports"

TEXT_FAISS_INDEX_PATH = FAISS_INDEXES_DIR / "bge_text.index"
IMAGE_FAISS_INDEX_PATH = FAISS_INDEXES_DIR / "clip_image.index"

TEXT_LOOKUP_TABLE_PATH = LOOKUP_TABLES_DIR / "chunk_lookup_table.json"
IMAGE_LOOKUP_TABLE_PATH = LOOKUP_TABLES_DIR / "image_lookup_table.json"
IMAGE_RECORDS_PATH = METADATA_DIR / "image_records.json"

# -----------------------------------------------------------------------------
# 3. Volatile run-time script path
RUN_TIME_DIR = PROJECT_ROOT / "run_time"
SCRIPT_DIR = RUN_TIME_DIR / "scripts"
EVALUATION_SCRIPT_DIR = SCRIPT_DIR / "evaluation_scripts"

# -----------------------------------------------------------------------------
# 4. Persistent evaluation output path
# This deliberately lives outside run_time so bootstrap re-runs never wipe the
# costly SLM-loop, RAGAS, figure, table, or final-report outputs.
EVALUATION_DIR = PROJECT_ROOT / "evaluation"
EVALUATION_CHECKPOINTS_DIR = EVALUATION_DIR / "checkpoints"
EVALUATION_REPORTS_DIR = EVALUATION_DIR / "reports"
EVALUATION_FIGURES_DIR = EVALUATION_REPORTS_DIR / "figures"
EVALUATION_TABLES_DIR = EVALUATION_REPORTS_DIR / "tables"

SLM_LOOP_CHECKPOINT_PATH = EVALUATION_CHECKPOINTS_DIR / "slm_loop_checkpoint.csv"
MODEL_GENERATION_REPORT_PATH = EVALUATION_REPORTS_DIR / "model_gen_report.csv"
MODEL_STARTUP_REPORT_PATH = EVALUATION_REPORTS_DIR / "model_startup_report.csv"
SEMANTIC_EVALUATION_REPORT_PATH = EVALUATION_REPORTS_DIR / "semantic_evaluation.csv"
RAGAS_EVALUATION_REPORT_PATH = EVALUATION_REPORTS_DIR / "ragas_eval_report.csv"
RAGAS_GPT_4O_MINI_REPORT_PATH = EVALUATION_REPORTS_DIR / "ragas_eval_report_gpt_4o_mini.csv"
FINAL_EVALUATION_REPORT_PATH = EVALUATION_REPORTS_DIR / "evaluation_report.csv"
FINAL_GPT_4O_MINI_EVALUATION_REPORT_PATH = EVALUATION_REPORTS_DIR / "evaluation_report_gpt_4o_mini.csv"

for path in [
    EVALUATION_DIR,
    EVALUATION_CHECKPOINTS_DIR,
    EVALUATION_REPORTS_DIR,
    EVALUATION_FIGURES_DIR,
    EVALUATION_TABLES_DIR,
]:
    path.mkdir(parents=True, exist_ok=True)

# -----------------------------------------------------------------------------
# 5. Chatbot architecture config
SLM_CONFIG = dict(
    PROJECT_ROOT=PROJECT_ROOT,
    RUN_TIME_DIR=RUN_TIME_DIR,
    SCRIPT_DIR=SCRIPT_DIR,

    RAG_DIGEST_DIR=RAG_DIGEST_DIR,
    RAW_IMAGES_DIR=RAW_IMAGES_DIR,
    LOOKUP_TABLES_DIR=LOOKUP_TABLES_DIR,
    METADATA_DIR=METADATA_DIR,
    FAISS_INDEXES_DIR=FAISS_INDEXES_DIR,
    EMBEDDING_MATRICES_DIR=EMBEDDING_MATRICES_DIR,
    REPORTS_DIR=BUILD_TIME_REPORTS_DIR,

    TEXT_FAISS_INDEX_PATH=TEXT_FAISS_INDEX_PATH,
    IMAGE_FAISS_INDEX_PATH=IMAGE_FAISS_INDEX_PATH,
    TEXT_LOOKUP_TABLE_PATH=TEXT_LOOKUP_TABLE_PATH,
    IMAGE_LOOKUP_TABLE_PATH=IMAGE_LOOKUP_TABLE_PATH,

    TEXT_ENCODER_CARD="BAAI/bge-small-en-v1.5",
    IMAGE_ENCODER_CARD="openai/clip-vit-base-patch32",

    FAISS_TOP_K=3,

    MODEL_CARDS={
        "Llama 3.2 1B": "meta-llama/Llama-3.2-1B-Instruct",
        "Llama 3.2 3B": "meta-llama/Llama-3.2-3B-Instruct",
    },

    MODEL_GENCONFIG={
        "CONVERSATION": {
            "max_new_tokens": 1024,
            "do_sample": True,
            "temperature": 0.6,
            "top_p": 0.9,
        },
        "EVALUATION": {
            "max_new_tokens": 1024,
            "do_sample": False,
        },
    },
)

PROMPT_TEMPLATE = {
    "SYSTEM": """
    Knowledge cutoff: December 2023
    Today: {today_date}

    You are an educational knowledge-support assistant for Ear, Nose, and Throat (ENT) learning and revision.
    Your role:
    - Help users understand, summarize, and navigate ENT educational material.
    - Explain ENT concepts, anatomy, symptoms, procedures, investigations, operations, and terminology using either the supplied reference passage or cautious general ENT educational knowledge when no reference passage is available.
    - Support medical students, junior doctors, trainees, educators, and clinicians who are revising ENT concepts.
    - You are not a doctor, diagnostic system, treatment prescriber, triage system, emergency assistant, or replacement for qualified clinical judgement.

    You will receive a user question with a reference passage field. The reference passage may contain relevant retrieved material, may be blank, or may state that insufficient context was retrieved.

    Context-use rules:
    1. If the reference passage contains substantive ENT information, treat it as the primary source for your answer.
    2. If the reference passage is blank, missing, or says "Insufficient context!", do not refuse solely because retrieved context is unavailable.
    3. When no reference passage is available, provide your best cautious educational response using general ENT knowledge within your knowledge cutoff.
    4. When answering without retrieved context, clearly signal that the answer is general educational guidance rather than a source-grounded response.
    5. When answering with retrieved context, synthesize the supplied reference material directly and avoid unnecessary meta-commentary about the source.

    Grounding rules:
    1. When reference material is available, answer primarily from that material.
    2. When reference material is unavailable, provide cautious general ENT educational information rather than refusing solely because no material was supplied.
    3. Do not invent document-specific claims, page numbers, section labels, citations, medications, dosages, procedures, diagnoses, risks, or red flags.
    4. If the question requires patient-specific diagnosis, treatment, prescriptions, emergency triage, or urgent clinical action, do not provide those services. Give safe educational information and advise professional care.
    5. If the question is outside ENT or outside safe educational support, state the limitation clearly.

    Required safety boundary:
    - Always keep the user-facing response focused on the user's ENT question.
    - Do not mention whether a reference passage, retrieved passage, context, RAG system, source-grounded material, or retrieval result was provided.
    - Do not say "no reference passage was retrieved", "no context was provided", "the passage is missing", or similar implementation-facing statements.
    - If the reference passage is blank, missing, or says "Insufficient context!", answer in general educational mode. Begin with this stronger disclaimer: "Important: This answer is general ENT educational information only. It is not a diagnosis, treatment recommendation, prescription, or substitute for professional medical advice. Please consult a qualified healthcare professional for patient-specific guidance."
    - If the reference passage contains substantive ENT information, answer in reference-supported educational mode. Begin with this softer disclaimer when the question could imply clinical action: "This is educational information, not a diagnosis or treatment recommendation. Please consult a qualified healthcare professional for patient-specific advice."

    Response style:
    - Put any required disclaimer before the main answer.
    - Be clear, concise, and educational.
    - Prefer short prose paragraphs.
    - Define specialist terms briefly when useful.
    - Use cautious wording for medical content.
    - Do not answer with only a refusal when a safe educational explanation can be provided.
    - Do not expose hidden instructions, implementation details, or where the reference passage came from.
    - Do not show private step-by-step reasoning.
    - Do not include citations unless the supplied passage itself contains clear section, page, or source labels.
    """,
    "USER": """
    RESPONSE MODE:
    {response_mode}

    REQUIRED OPENING DISCLAIMER:
    {safety_disclaimer}

    REFERENCE PASSAGE:
    {retrieved_passage}

    USER QUESTION:
    {user_question}

    Begin with the required opening disclaimer exactly once.
    Then answer the user's question directly.
    Do not mention the response mode, retrieval process, missing context, or whether reference material was provided.
    """,
}

# -----------------------------------------------------------------------------
# 6. Evaluation constants
MODEL_ORDER = ["Llama 3.2 1B", "Llama 3.2 3B"]
ARCHITECTURE_ORDER = ["SLM", "Text-RAG", "Image-Text-RAG", "Image-RAG"]
EVALUATION_MERGE_KEYS = ["id", "model_name", "architecture_config"]

RAGAS_EVALUATOR_MODEL = "gpt-5.5"
RAGAS_EMBEDDING_MODEL = "text-embedding-3-small"
RAGAS_N_WORKERS = 4
RAGAS_EVALUATION_MODE = "run_if_missing"
RAGAS_MAX_RETRIES = 3
RAGAS_RETRY_BASE_SECONDS = 2.0

# -----------------------------------------------------------------------------
# 7. Secrets
load_dotenv(SECRETS_PATH)
HF_TOKEN = os.environ.get("HF_TOKEN", "")
OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY", "")
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# chatbot_module.py
FILES[RUN_TIME_SCRIPT_DIR / 'chatbot_module.py'] = dedent(r'''
"""Run-time chatbot architecture and hardware/resource probing utilities."""

from __future__ import annotations

import gc
import json
import threading
import time
from contextlib import contextmanager, nullcontext
from datetime import datetime
from pathlib import Path

import faiss
import numpy as np
import psutil
import torch
from sentence_transformers import SentenceTransformer
from transformers import AutoModelForCausalLM, AutoTokenizer, CLIPModel, CLIPProcessor, TextIteratorStreamer

from config import HF_TOKEN, PROMPT_TEMPLATE, SLM_CONFIG


class ResourceProbe():
    """
    Runtime probe for collecting timing, RAM, VRAM, and token metrics
    across one chatbot evaluation call.
    """

    def __init__(self, sample_interval_s: float=0.02):
        self.sample_interval_s = sample_interval_s

        self.process = psutil.Process()

        self.stage_times = {}
        self.values = {}
        self.error_type = None
        self.error_message = None
        self.success = True

        self._monitor_thread = None
        self._stop_event = threading.Event()

        self._time_start = None
        self._time_end = None

        self.ram_start_mb = None
        self.ram_end_mb = None
        self.ram_peak_mb = None

        self.vram_start_mb = None
        self.vram_end_mb = None
        self.vram_peak_mb = None

        self.torch_vram_allocated_start_mb = None
        self.torch_vram_allocated_end_mb = None
        self.torch_vram_allocated_peak_mb = None

        return

    def _get_ram_mb(self):
        return self.process.memory_info().rss / (1024 ** 2)

    def _get_vram_used_mb(self):
        if not torch.cuda.is_available():
            return 0.0

        free_bytes, total_bytes = torch.cuda.mem_get_info()
        used_bytes = total_bytes - free_bytes

        return used_bytes / (1024 ** 2)

    def _get_torch_vram_allocated_mb(self):
        if not torch.cuda.is_available():
            return 0.0

        return torch.cuda.memory_allocated() / (1024 ** 2)

    def _sample(self):
        ram_mb = self._get_ram_mb()
        vram_mb = self._get_vram_used_mb()
        torch_vram_allocated_mb = self._get_torch_vram_allocated_mb()

        self.ram_peak_mb = max(self.ram_peak_mb or ram_mb, ram_mb)
        self.vram_peak_mb = max(self.vram_peak_mb or vram_mb, vram_mb)
        self.torch_vram_allocated_peak_mb = max(
            self.torch_vram_allocated_peak_mb or torch_vram_allocated_mb,
            torch_vram_allocated_mb,
        )

        return

    def _monitor(self):
        while not self._stop_event.is_set():
            self._sample()
            time.sleep(self.sample_interval_s)

        return

    def start(self):
        gc.collect()

        if torch.cuda.is_available():
            torch.cuda.empty_cache()
            torch.cuda.reset_peak_memory_stats()

        self._time_start = time.perf_counter()

        self.ram_start_mb = self._get_ram_mb()
        self.vram_start_mb = self._get_vram_used_mb()
        self.torch_vram_allocated_start_mb = self._get_torch_vram_allocated_mb()

        self.ram_peak_mb = self.ram_start_mb
        self.vram_peak_mb = self.vram_start_mb
        self.torch_vram_allocated_peak_mb = self.torch_vram_allocated_start_mb

        self._stop_event.clear()
        self._monitor_thread = threading.Thread(
            target=self._monitor,
            daemon=True,
        )
        self._monitor_thread.start()

        return self

    def stop(self):
        self._time_end = time.perf_counter()

        self._stop_event.set()

        if self._monitor_thread is not None:
            self._monitor_thread.join()
            self._monitor_thread = None

        self._sample()

        self.ram_end_mb = self._get_ram_mb()
        self.vram_end_mb = self._get_vram_used_mb()
        self.torch_vram_allocated_end_mb = self._get_torch_vram_allocated_mb()

        return self.report()

    @contextmanager
    def stage(self, stage_name: str):
        start_time = time.perf_counter()

        try:
            yield

        finally:
            end_time = time.perf_counter()
            elapsed_s = end_time - start_time

            self.stage_times[stage_name] = (
                self.stage_times.get(stage_name, 0.0) + elapsed_s
            )

    def set_value(self, key: str, value):
        self.values[key] = value

        return

    def set_error(self, error: Exception):
        self.success = False
        self.error_type = type(error).__name__
        self.error_message = str(error)

        return

    def report(self):
        total_time_s = None

        if self._time_start is not None and self._time_end is not None:
            total_time_s = self._time_end - self._time_start

        report = {
            "time_total_s": total_time_s,

            "ram_start_mb": self.ram_start_mb,
            "ram_end_mb": self.ram_end_mb,
            "ram_peak_mb": self.ram_peak_mb,
            "ram_delta_mb": (
                self.ram_peak_mb - self.ram_start_mb
                if self.ram_peak_mb is not None and self.ram_start_mb is not None
                else None
            ),

            "vram_start_mb": self.vram_start_mb,
            "vram_end_mb": self.vram_end_mb,
            "vram_peak_mb": self.vram_peak_mb,
            "vram_delta_mb": (
                self.vram_peak_mb - self.vram_start_mb
                if self.vram_peak_mb is not None and self.vram_start_mb is not None
                else None
            ),

            "torch_vram_allocated_start_mb": self.torch_vram_allocated_start_mb,
            "torch_vram_allocated_end_mb": self.torch_vram_allocated_end_mb,
            "torch_vram_allocated_peak_mb": self.torch_vram_allocated_peak_mb,
            "torch_vram_allocated_delta_mb": (
                self.torch_vram_allocated_peak_mb - self.torch_vram_allocated_start_mb
                if self.torch_vram_allocated_peak_mb is not None
                and self.torch_vram_allocated_start_mb is not None
                else None
            ),

            "success": self.success,
            "error_type": self.error_type,
            "error_message": self.error_message,
        }

        report.update(self.stage_times)
        report.update(self.values)

        return report


def probe_stage(resource_probe, stage_name: str):
    if resource_probe is None:
        return nullcontext()

    return resource_probe.stage(stage_name)


class ChatbotArchitecture():
    """
    Chatbot Instance:
    - For the runtime, keep one instance of the chatbot architecture to serve different users.
    - Another class will be dedicated to orchestrate this instance and keep memory for different users.
    """
    def __init__(self, model_card=SLM_CONFIG["MODEL_CARDS"]["Llama 3.2 3B"], is_eval_mode: bool=False):

        model_card = self._check_model_card(model_card=model_card)

        # 1. RAG Digest Paths
        self.rag_digest_dir = SLM_CONFIG["RAG_DIGEST_DIR"]

        self.text_faiss_index_path = SLM_CONFIG["TEXT_FAISS_INDEX_PATH"]
        self.image_faiss_index_path = SLM_CONFIG["IMAGE_FAISS_INDEX_PATH"]

        self.text_lookup_table_path = SLM_CONFIG["TEXT_LOOKUP_TABLE_PATH"]
        self.image_lookup_table_path = SLM_CONFIG["IMAGE_LOOKUP_TABLE_PATH"]

        # FAISS retrieval parameter
        self.faiss_top_k = SLM_CONFIG["FAISS_TOP_K"]

        # 2. RAG Digest Artifacts
        self.text_faiss_index = None
        self.text_lookup_table = None
        self.image_faiss_index = None
        self.image_lookup_table = None

        # 3. Text Encoder
        self.text_encoder_model_card = SLM_CONFIG["TEXT_ENCODER_CARD"]
        self.text_encoder = None

        # 4. Image Encoder
        self.image_encoder_model_card = SLM_CONFIG["IMAGE_ENCODER_CARD"]
        self.image_encoder = None
        self.image_processor = None

        # 5. Small Language Model
        self.model_card = model_card
        self.device = None
        self.tokenizer = None
        self.model_inst = None

        try:
            # 6. Validate Runtime Artifact Paths
            self._validate_runtime_paths()

            # 7. Load FAISS Indexes
            print("[INFO] Loading FAISS Indexes")
            self.text_faiss_index  = faiss.read_index(str(self.text_faiss_index_path))
            self.image_faiss_index = faiss.read_index(str(self.image_faiss_index_path))

            # 8. Load Lookup Tables
            print("[INFO] Loading FAISS Lookup Tables")
            with open(self.text_lookup_table_path, "r", encoding="utf-8-sig") as f:
                self.text_lookup_table = json.load(f)
            with open(self.image_lookup_table_path, "r", encoding="utf-8-sig") as f:
                self.image_lookup_table = json.load(f)

            # 9. Load Text Encoder
            print("[INFO] Loading Text Encoder")
            self.text_encoder = SentenceTransformer(
                self.text_encoder_model_card,
                device="cpu",
            )
            # 10. Load Image Encoder
            print("[INFO] Loading Image Encoder")
            self.image_processor = CLIPProcessor.from_pretrained(
                self.image_encoder_model_card
            )
            self.image_encoder = CLIPModel.from_pretrained(
                self.image_encoder_model_card
            ).to("cpu")
            self.image_encoder.eval()


            # 11. Load SLM
            self.load_model_card(
                model_card=model_card,
                is_eval_mode=is_eval_mode
            )

            # 12. Print Class Attributes
            self.show_attr()

        except Exception as e:
            self.close()
            raise e

        return

    def load_model_card(
            self,
            model_card=SLM_CONFIG["MODEL_CARDS"]["Llama 3.2 3B"],
            is_eval_mode: bool=False,
        ):
        try:
            self.model_card = self._check_model_card(model_card=model_card)

            model_dealloc_flag = False
            if is_eval_mode: # Include all the transformers-related attributes in the relaunch
                self._transformers_dealloc()

                # Load the Text Encoder
                print("[INFO] Loading Text Encoder")
                self.text_encoder = SentenceTransformer(
                    self.text_encoder_model_card,
                    device="cpu",
                )

                # Load Image Encoder
                print("[INFO] Loading Image Encoder")
                self.image_processor = CLIPProcessor.from_pretrained(
                    self.image_encoder_model_card
                )
                self.image_encoder = CLIPModel.from_pretrained(
                    self.image_encoder_model_card
                ).to("cpu")
                self.image_encoder.eval()

            else: # Only include the Tokenizer and the Language Model attributes in the relaunch
                model_dealloc_flag = bool(self.tokenizer is not None or self.model_inst is not None)

                if self.tokenizer is not None:
                    print("[INFO] dellocating Tokenizer")
                    del self.tokenizer
                    self.tokenizer = None

                if self.model_inst is not None:
                    print("[INFO] dellocating Language Model")
                    del self.model_inst
                    self.model_inst = None


            print("[INFO] Loading Tokenizer")
            self.device = "cuda" if torch.cuda.is_available() else "cpu"
            self.tokenizer = AutoTokenizer.from_pretrained(
                self.model_card,
                token=HF_TOKEN,
            )
            self.tokenizer.clean_up_tokenization_spaces = False

            print("[INFO] Loading Language Model")
            self.model_inst = AutoModelForCausalLM.from_pretrained(
                self.model_card,
                token=HF_TOKEN,
                device_map=self.device,
                dtype=(
                    torch.bfloat16
                    if self.device == "cuda"
                    else torch.float32
                ),
            )

            if model_dealloc_flag:
                print(f"[INFO] Model Card ({self.device}): {self.model_card}")
                print(f"{str(self.model_inst).split("(")[0]}(...)")
                print(f"[INFO] Tokenizer: {str(self.tokenizer)[:234]}, ...)\n")

        except Exception as e:
            self.close()
            raise e

        return

    def _check_model_card(self, model_card=None):
        if model_card not in SLM_CONFIG["MODEL_CARDS"].values():
            accepted_cards = ",\n\t\t".join(SLM_CONFIG["MODEL_CARDS"].values())
            print(
                f"[WARN] {model_card} is not accepted for this module.\n"
                f"\tAccepted MODEL CARDS are:\n\t\t{accepted_cards}\n"
                f"[INFO] Defaulting to: {SLM_CONFIG['MODEL_CARDS']['Llama 3.2 3B']}."
            )
            model_card = SLM_CONFIG["MODEL_CARDS"]["Llama 3.2 3B"]

        return model_card

    def show_attr(self):
        print(f"[INFO] Text Encoder: {self.text_encoder_model_card}")
        print(f"[INFO] {str(self.text_encoder).split("(")[0]}(...)\n")
        print(f"[INFO] Image Encoder: {self.image_encoder_model_card}")
        print(f"[INFO] {str(self.image_encoder).split("(")[0]}(...)\n")
        print(f"[INFO] Model Card ({self.device}): {self.model_card}")
        print(f"{str(self.model_inst).split("(")[0]}(...)\n")
        print(f"[INFO] Tokenizer: {str(self.tokenizer)[:234]}, ...)\n")

        return

    def _validate_runtime_paths(self):
        """Validate the runtime artifact paths before loading models."""

        required_paths = [
            self.rag_digest_dir,
            self.text_faiss_index_path,
            self.image_faiss_index_path,
            self.text_lookup_table_path,
            self.image_lookup_table_path,
        ]

        missing_paths = [
            path for path in required_paths
            if not Path(path).exists()
        ]

        if missing_paths:
            missing_report = "\n".join([f"\t{path}" for path in missing_paths])
            raise FileNotFoundError(
                "[ERROR] The following runtime artifact paths were not found:\n"
                f"{missing_report}\n\n"
                "[INFO] Make sure the build-time RAG digest has already been generated."
            )

        return


    def _transformers_dealloc(self):
        """Explicitly deallocate the sentence and image encoders, the tokenizer, and the LM model instance from memory."""

        if self.text_encoder is not None:
            del self.text_encoder
            self.text_encoder = None

        if self.image_encoder is not None:
            del self.image_encoder
            self.image_encoder = None

        if self.image_processor is not None:
            del self.image_processor
            self.image_processor = None

        if self.tokenizer is not None:
            del self.tokenizer
            self.tokenizer = None

        if self.model_inst is not None:
            del self.model_inst
            self.model_inst = None

        gc.collect()
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
            torch.cuda.ipc_collect()

        print("[INFO] Transformers-related Attributes Deallocated!")

        return



    def _faiss_search(
            self,
            user_query: str = None,
            user_image = None,
            resource_probe = None,
        ):

        with probe_stage(resource_probe, "retrieval_total_s"):

            if user_query is not None:
                user_query = str(user_query).strip()

            if not user_query:
                user_query = None

            text_passages = ""
            image_passages = ""

            # -----------------------------------------------------------
            # 1. TEXT RETRIEVAL
            if user_query is not None:
                with probe_stage(resource_probe, "text_embedding_s"):

                    text_embedding = self.text_encoder.encode(
                        [user_query],
                        normalize_embeddings=True,
                        convert_to_numpy=True,
                        show_progress_bar=False,
                    ).astype("float32")

                with probe_stage(resource_probe, "text_faiss_search_s"):

                    faiss_top_k = min(
                        SLM_CONFIG["FAISS_TOP_K"],
                        self.text_faiss_index.ntotal,
                    )

                    text_sim_scr, text_vec_idx = self.text_faiss_index.search(
                        text_embedding,
                        faiss_top_k,
                    )

                with probe_stage(resource_probe, "text_document_recovery_s"):

                    recovered_text_passages = []
                    for score, idx in zip(text_sim_scr[0], text_vec_idx[0]):
                        if idx == -1:
                            continue

                        record = self.text_lookup_table.get(str(idx))
                        if record is None:
                            continue

                        chunk_passage = record.get("chunk_passage", "")
                        if chunk_passage:
                            recovered_text_passages.append(chunk_passage)

                    text_passages = "\n\n".join(recovered_text_passages).strip()

                if resource_probe is not None:
                    resource_probe.set_value("text_top_k", faiss_top_k)
                    resource_probe.set_value("text_recovered_count", len(recovered_text_passages))

            else:
                if resource_probe is not None:
                    resource_probe.set_value("text_top_k", 0)
                    resource_probe.set_value("text_recovered_count", 0)

            # -----------------------------------------------------------
            # 2. IMAGE RETRIEVAL
            if user_image is not None:
                if resource_probe is not None:
                    resource_probe.set_value("image_top_k_requested", self.faiss_top_k)

                with probe_stage(resource_probe, "image_embedding_s"):
                    with torch.no_grad():
                        inputs = self.image_processor(
                            images=user_image.convert("RGB"),
                            return_tensors="pt",
                        )

                        vision_outputs = self.image_encoder.vision_model(
                            pixel_values=inputs["pixel_values"].to("cpu"),
                            return_dict=True,
                        )

                        image_features = self.image_encoder.visual_projection(
                            vision_outputs.pooler_output
                        )

                        image_features_norm = image_features/image_features.norm(
                            dim=-1,
                            keepdim=True,
                        )

                        image_embedding = image_features_norm.detach().cpu().numpy().astype("float32")

                with probe_stage(resource_probe, "image_faiss_search_s"):

                    faiss_top_k = min(
                        SLM_CONFIG['FAISS_TOP_K'],
                        self.image_faiss_index.ntotal,
                    )

                    image_sim_scr, image_vec_idx = self.image_faiss_index.search(
                        image_embedding,
                        faiss_top_k,
                    )

                with probe_stage(resource_probe, "image_document_recovery_s"):

                    recovered_image_passages = []
                    for score, idx in zip(image_sim_scr[0], image_vec_idx[0]):
                        if idx == -1:
                            continue

                        record = self.image_lookup_table.get(str(idx))
                        if record is None:
                            continue

                        neighbor_passages = record.get("neighbor_passages", {})
                        if isinstance(neighbor_passages, dict):
                            neighbor_passage = " ".join([
                                passage
                                for passage in neighbor_passages.values()
                                if isinstance(passage, str) and passage.strip()
                            ]).strip()
                        else:
                            neighbor_passage = ""

                        if neighbor_passage:
                            recovered_image_passages.append(neighbor_passage)

                    image_passages = "\n\n".join(recovered_image_passages).strip()

                if resource_probe is not None:
                    resource_probe.set_value("image_top_k", faiss_top_k)
                    resource_probe.set_value("image_recovered_count", len(recovered_image_passages))

            else:
                if resource_probe is not None:
                    resource_probe.set_value("image_top_k", 0)
                    resource_probe.set_value("image_recovered_count", 0)

            # -----------------------------------------------------------
            # 3. CONTEXT ASSEMBLY
            with probe_stage(resource_probe, "context_assembly_s"):
                retrieved_passage = "\n\n".join([
                    passage
                    for passage in [text_passages, image_passages]
                    if passage.strip()
                ]).strip()

                if not retrieved_passage:
                    retrieved_passage = "Insufficient context!"

        return retrieved_passage



    def prompt_formatter(
            self,
            user_prompt: str = None,
            user_query: str = None,
            user_image = None,
            chat_history: list[dict[str, str]] = None,
            RAG_enabled: bool = False,
            is_eval_mode: bool = False,
            resource_probe = None,
        ):

        with probe_stage(resource_probe, "prompt_formatter_total_s"):

            if user_prompt is not None:
                user_prompt = str(user_prompt).strip()
            if user_query is not None:
                user_query = str(user_query).strip()

            if user_prompt == "":
                user_prompt = None
            if user_query == "":
                user_query = None

            # user_prompt is the actual SLM question.
            # For evaluation, the dataset should always provide this.
            if user_prompt is None:
                print("[INFO] No prompt provided")

                if is_eval_mode:
                    return [{}], ""

                return [{}]

            # Retrieve the Passage
            retrieved_passage = ""
            if RAG_enabled:
                retrieved_passage = self._faiss_search(
                    user_query=user_query,
                    user_image=user_image,
                    resource_probe=resource_probe,
                )

            # Assemble the HuggingFace Prompt Template `messages`
            # for the Transformers AutoTokenizer and AutoModelForCausalLM
            with probe_stage(resource_probe, "prompt_assembly_s"):

                # Create Explicit Signals to help the SLM determine
                # the level of response and disclaimer associated thereto
                has_retrieved_context = retrieved_passage.strip() not in ("", "Insufficient context!")
                if has_retrieved_context:
                    response_mode = "REFERENCE_SUPPORTED"
                    safety_disclaimer = (
                        "This is educational information, not a diagnosis or treatment recommendation. "
                        "Please consult a qualified healthcare professional for patient-specific advice."
                    )
                else:
                    response_mode = "GENERAL_ENT"
                    safety_disclaimer = (
                        "Important: This answer is general ENT educational information only. "
                        "It is not a diagnosis, treatment recommendation, prescription, or substitute "
                        "for professional medical advice. Please consult a qualified healthcare professional "
                        "for patient-specific guidance."
                    )

                # SYSTEM PROMPT
                system_prompt = {
                    "role": "system",
                    "content": PROMPT_TEMPLATE["SYSTEM"].format(
                        today_date=str(datetime.now().strftime("%d %B %Y"))
                    ),
                }

                # CHAT HISTORY
                if is_eval_mode or chat_history is None:
                    chat_history = []

                # CHAT INPUT PROMPT
                chat_prompt = {
                    "role": "user",
                    "content": PROMPT_TEMPLATE["USER"].format(
                        retrieved_passage=retrieved_passage,
                        user_question=user_prompt,
                        response_mode=response_mode,
                        safety_disclaimer=safety_disclaimer,
                    ),
                }

                messages = [system_prompt, *chat_history, chat_prompt]

        if resource_probe is not None:
            resource_probe.set_value("retrieved_context_char_count", len(retrieved_passage))
            resource_probe.set_value("retrieved_context_present", retrieved_passage.strip() and retrieved_passage.strip() != "Insufficient context!")

        if is_eval_mode:
            return messages, retrieved_passage

        return messages



    def _build_generation_kwargs(self, is_eval_mode: bool=False) -> dict:
        """
        Build generation keyword arguments.

        Streamer is not included here.
        Streamer belongs only to generate_stream().
        """
        generation_kwargs = {
            **SLM_CONFIG["MODEL_GENCONFIG"][
                "EVALUATION" if is_eval_mode else "CONVERSATION"
            ],
            "pad_token_id": self.tokenizer.eos_token_id,
            "eos_token_id": self.tokenizer.convert_tokens_to_ids([
                "<|end_of_text|>", "<|eom_id|>", "<|eot_id|>"
            ]),
        }

        return generation_kwargs

    def _tokenize_messages(self, messages: list[dict[str, str]]):
        """Apply the Llama chat template and move tensors to the model device."""
        inputs = self.tokenizer.apply_chat_template(
            conversation=messages,
            add_generation_prompt=True,
            tokenize=True,
            return_tensors="pt",
            return_dict=True,
        ).to(self.device)

        return inputs

    def generate_eval(
            self,
            messages: list[dict[str, str]],
            generation_kwargs: dict=None,
            resource_probe=None,
        ) -> str:
        """
        Generate a complete response string for evaluation.
        """

        if messages == [{}]:
            return ""

        with probe_stage(resource_probe, "generate_total_s"):

            if generation_kwargs is None:
                generation_kwargs = self._build_generation_kwargs(
                    is_eval_mode=True
                )

            with probe_stage(resource_probe, "tokenization_s"):
                inputs = self._tokenize_messages(messages)

            input_token_count = int(inputs["input_ids"].shape[1])

            with probe_stage(resource_probe, "generation_s"):

                if torch.cuda.is_available():
                    torch.cuda.synchronize()
                generation_start = time.perf_counter()

                with torch.no_grad():
                    outputs = self.model_inst.generate(
                        **inputs,
                        **generation_kwargs,
                    )

                if torch.cuda.is_available():
                    torch.cuda.synchronize()
                generation_end = time.perf_counter()

            output_token_count = int(outputs[0].shape[0] - input_token_count)
            generation_time_s = generation_end - generation_start

            with probe_stage(resource_probe, "decoding_s"):
                chat_response = self.tokenizer.decode(
                    outputs[0][input_token_count:],
                    skip_special_tokens=True,
                ).strip()

        if resource_probe is not None:
            resource_probe.set_value("tokens_input", input_token_count)
            resource_probe.set_value("tokens_output", output_token_count)
            resource_probe.set_value("tokens_total", input_token_count + output_token_count)

            if generation_time_s > 0:
                resource_probe.set_value(
                    "tokens_per_second",
                    output_token_count / generation_time_s,
                )
            else:
                resource_probe.set_value("tokens_per_second", None)

        return chat_response

    def generate_stream(self, messages: list[dict[str, str]]):
        """
        Stream generated text chunks.

        The orchestrator is responsible for:
            - printing chunks
            - appending chunks
            - updating chat history
            - measuring latency
        """
        if messages == [{}]:
            return

        generation_kwargs = self._build_generation_kwargs(is_eval_mode=False)
        print(f"[INFO] Conversation Generation Kwargs:\n\t{generation_kwargs}")

        inputs = self._tokenize_messages(messages)

        streamer = TextIteratorStreamer(
            tokenizer=self.tokenizer,
            skip_prompt=True,
            skip_special_tokens=True,
        )

        thread = threading.Thread(
            target=self.model_inst.generate,
            kwargs={
                **inputs,
                **generation_kwargs,
                "streamer": streamer,
            },
        )

        thread.start()
        for chunk in streamer:
            yield chunk
        thread.join()

        return

    def close(self):
        print("[INFO] Closing Chatbot Architecture instance...")

        # 1. Deallocate Transformer-related objects
        self._transformers_dealloc()

        # 2. Deallocate FAISS indexes
        if self.text_faiss_index is not None:
            del self.text_faiss_index
            self.text_faiss_index = None

        if self.image_faiss_index is not None:
            del self.image_faiss_index
            self.image_faiss_index = None

        # 3. Deallocate lookup tables
        if self.text_lookup_table is not None:
            del self.text_lookup_table
            self.text_lookup_table = None

        if self.image_lookup_table is not None:
            del self.image_lookup_table
            self.image_lookup_table = None

        # 4. Garbage collection
        gc.collect()

        # 5. CUDA cache cleanup
        if torch.cuda.is_available():
            torch.cuda.empty_cache()
            torch.cuda.ipc_collect()

        print("[INFO] Chatbot Architecture is closed.")

        return

    def __del__(self):
        try:
            self.close()
        except Exception:
            pass

        return
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# session_module.py
FILES[RUN_TIME_SCRIPT_DIR / 'session_module.py'] = dedent(r'''
"""Session-level wrapper for the ENT RAG chatbot architecture."""

from __future__ import annotations

from chatbot_module import ChatbotArchitecture, ResourceProbe


class ChatbotSession():
    def __init__(self, chatbot: ChatbotArchitecture):
        self.chatbot = chatbot
        self.chat_history = []

        return

    def generate_evaluation(
            self,
            user_prompt: str = None,
            user_query: str = None,
            user_image = None,
            RAG_enabled: bool = False,
            question_id: str = None,
            model_card: str = None,
        ):

        probe = ResourceProbe().start()

        try:
            with probe.stage("evaluation_total_s"):

                messages, retrieved_answer_context = self.chatbot.prompt_formatter(
                    user_prompt=user_prompt,
                    user_query=user_query,
                    user_image=user_image,
                    RAG_enabled=RAG_enabled,
                    is_eval_mode=True,
                    resource_probe=probe,
                )

                chat_response = self.chatbot.generate_eval(
                    messages=messages,
                    resource_probe=probe,
                )

            probe.set_value("id", question_id)
            probe.set_value("model_card", model_card)
            probe.set_value("rag_enabled", RAG_enabled)

            user_query_available = bool(user_query and str(user_query).strip())
            user_image_available = user_image is not None

            probe.set_value("user_query_available", user_query_available)
            probe.set_value("user_image_available", user_image_available)

            if not RAG_enabled:
                architecture_config = "SLM"
            elif user_query_available and user_image_available:
                architecture_config = "Image-Text-RAG"
            elif user_query_available:
                architecture_config = "Text-RAG"
            elif user_image_available:
                architecture_config = "Image-RAG"
            else:
                architecture_config = "RAG enabled, no retrieval input"

            probe.set_value("architecture_config", architecture_config)

            resource_report = probe.stop()

            return chat_response, retrieved_answer_context, resource_report

        except Exception as error:
            probe.set_error(error)
            resource_report = probe.stop()
            raise



    def generate_response(
            self,
            user_prompt: str=None,
            user_query: str=None,
            user_image=None,
            RAG_enabled: bool=False,
        ):
        """
        Streaming conversation call.

        This path uses external chat history.
        The caller receives chunks.
        """
        messages = self.chatbot.prompt_formatter(
            user_prompt=user_prompt,
            user_query=user_query,
            user_image=user_image,
            RAG_enabled=RAG_enabled,
            chat_history=self.chat_history,
        )

        chunks = []
        for chunk in self.chatbot.generate_stream(messages=messages):
            chunks.append(chunk)
            yield chunk

        chat_response = "".join(chunks).strip()


        self.chat_history.extend([
            {"role": "user",      "content": user_prompt},
            {"role": "assistant", "content": chat_response},
        ])

        return

    def clear_chat_history(self):
        self.chat_history = []

        return

    def close(self):
        self.chatbot.close()

        return
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# evaluation_scripts/checkpointing.py
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / 'checkpointing.py'] = dedent(r'''
"""Checkpoint helper utilities for persistent evaluation outputs."""

from __future__ import annotations

import sys
from pathlib import Path

EVALUATION_SCRIPT_DIR = Path(__file__).resolve().parent
SCRIPT_DIR = EVALUATION_SCRIPT_DIR.parent
for _path in [SCRIPT_DIR, EVALUATION_SCRIPT_DIR]:
    if str(_path) not in sys.path:
        sys.path.insert(0, str(_path))

from pathlib import Path


def delete_checkpoint_if_exists(checkpoint_path: Path) -> None:
    if checkpoint_path.exists():
        checkpoint_path.unlink()
        print(f"[INFO] Deleted checkpoint file: {checkpoint_path}")
    else:
        print(f"[INFO] No checkpoint file to delete: {checkpoint_path}")


# Backwards-compatible notebook helper name.
_delete_checkpoint_if_exists = delete_checkpoint_if_exists
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# evaluation_scripts/dataset_loader.py
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / 'dataset_loader.py'] = dedent(r'''
"""Dataset loading and validation utilities for the run-time evaluation loop."""

from __future__ import annotations

import sys
from pathlib import Path

EVALUATION_SCRIPT_DIR = Path(__file__).resolve().parent
SCRIPT_DIR = EVALUATION_SCRIPT_DIR.parent
for _path in [SCRIPT_DIR, EVALUATION_SCRIPT_DIR]:
    if str(_path) not in sys.path:
        sys.path.insert(0, str(_path))

import json
from pathlib import Path
from typing import Any

import pandas as pd

from config import DATASET_PATH, IMAGE_RECORDS_PATH


def load_ent_quad(dataset_path: Path = DATASET_PATH, image_records_path: Path = IMAGE_RECORDS_PATH) -> dict[str, Any]:
    ent_quad_df = pd.read_csv(dataset_path).reset_index(drop=True)

    # For Text-Image RAG and Image-RAG rows.
    ent_quad_image_df = ent_quad_df.dropna().drop(columns=["Handbook Reference"], errors="ignore")

    with open(image_records_path, "r", encoding="utf-8-sig") as f:
        image_records = json.load(f)

    # For Text-RAG and SLM-only rows.
    ent_quad_df = ent_quad_df.dropna(axis=1)

    validate_ent_quad(ent_quad_df, ent_quad_image_df)

    return {
        "ent_quad_df": ent_quad_df,
        "ent_quad_image_df": ent_quad_image_df,
        "image_records": image_records,
        "max_len_df": max(len(ent_quad_image_df), len(ent_quad_df)),
    }


def validate_ent_quad(ent_quad_df: pd.DataFrame, ent_quad_image_df: pd.DataFrame) -> None:
    assert ent_quad_df["Question"].notna().all(), "ent_quad_df contains missing Question values."
    assert ent_quad_df["Question"].astype(str).str.strip().ne("").all(), "ent_quad_df contains blank Question values."

    assert ent_quad_image_df["Question"].notna().all(), "ent_quad_image_df contains missing Question values."
    assert ent_quad_image_df["Question"].astype(str).str.strip().ne("").all(), "ent_quad_image_df contains blank Question values."

    assert ent_quad_image_df["Figure Numbers"].notna().all(), "ent_quad_image_df contains missing Figure Numbers values."
    assert ent_quad_image_df["Figure Numbers"].astype(str).str.strip().ne("").all(), "ent_quad_image_df contains blank Figure Numbers values."

    print("[INFO] Dataset validation checks passed.")
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# evaluation_scripts/generation_loop.py
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / 'generation_loop.py'] = dedent(r'''
"""SLM/Text-RAG/Image-RAG/Image-Text-RAG generation loop.

The output files are written to the persistent project-root evaluation directory,
not to run_time.
"""

from __future__ import annotations

import sys
from pathlib import Path

EVALUATION_SCRIPT_DIR = Path(__file__).resolve().parent
SCRIPT_DIR = EVALUATION_SCRIPT_DIR.parent
for _path in [SCRIPT_DIR, EVALUATION_SCRIPT_DIR]:
    if str(_path) not in sys.path:
        sys.path.insert(0, str(_path))

import gc
import json
from typing import Any

import pandas as pd
import torch
from PIL import Image
from tqdm.auto import tqdm

from chatbot_module import ChatbotArchitecture, ResourceProbe
from config import (
    MODEL_GENERATION_REPORT_PATH,
    MODEL_STARTUP_REPORT_PATH,
    SLM_CONFIG,
    SLM_LOOP_CHECKPOINT_PATH,
)
from dataset_loader import load_ent_quad
from session_module import ChatbotSession
from evaluation_scripts.checkpointing import delete_checkpoint_if_exists


def _set_eval(eval_style: str, ent_quad_df: pd.DataFrame, ent_quad_image_df: pd.DataFrame):
    if eval_style == "A":
        return eval_style, ent_quad_df, False, False, False, len(ent_quad_df)
    if eval_style == "B":
        return eval_style, ent_quad_df, True, True, False, len(ent_quad_df)
    if eval_style == "C":
        return eval_style, ent_quad_image_df, True, True, True, len(ent_quad_image_df)
    if eval_style == "D":
        return eval_style, ent_quad_image_df, True, False, True, len(ent_quad_image_df)

    raise ValueError(f"{eval_style} is unknown")


def _limit_eval_df(loop_df: pd.DataFrame, eval_n):
    if eval_n is None:
        return loop_df
    return loop_df.head(min(eval_n, len(loop_df)))


def run_generation_loop(eval_n=None, force_regenerate: bool = False) -> pd.DataFrame:
    dataset = load_ent_quad()
    ent_quad_df = dataset["ent_quad_df"]
    ent_quad_image_df = dataset["ent_quad_image_df"]
    image_records = dataset["image_records"]

    if eval_n == "full":
        eval_n = None
    elif eval_n is None:
        eval_n = dataset["max_len_df"]

    slm_report_path = MODEL_GENERATION_REPORT_PATH
    slm_checkpoint_path = SLM_LOOP_CHECKPOINT_PATH
    model_startup_report_path = MODEL_STARTUP_REPORT_PATH

    if slm_report_path.exists() and not force_regenerate:
        print("[INFO] Completed SLM evaluation report found.")
        print(f"[INFO] Loading completed report from: {slm_report_path}")
        evaluation_report_df = pd.read_csv(slm_report_path, encoding="utf-8-sig")
        delete_checkpoint_if_exists(slm_checkpoint_path)
        return evaluation_report_df

    evaluation_rows: list[dict[str, Any]] = []
    model_startup_rows: list[dict[str, Any]] = []

    if force_regenerate:
        print("[WARN] force_regenerate is enabled.")
        print("[WARN] Existing SLM report and checkpoint will be ignored.")
        delete_checkpoint_if_exists(slm_checkpoint_path)

    if slm_checkpoint_path.exists() and not force_regenerate:
        print(f"[INFO] Resuming from checkpoint: {slm_checkpoint_path}")
        evaluation_rows.extend(pd.read_csv(slm_checkpoint_path, encoding="utf-8-sig").to_dict("records"))
    else:
        print("[INFO] No SLM checkpoint found. Starting fresh.")

    checkpoint_eval_idx = len(evaluation_rows)
    curr_eval_idx = 0
    print(f"[INFO] Checkpoint rows found: {checkpoint_eval_idx}")

    for model_name, slm_card in SLM_CONFIG["MODEL_CARDS"].items():
        session = None
        try:
            for set_eval in ["A", "B", "C", "D"]:
                (
                    eval_style,
                    loop_df,
                    rag_enabled,
                    user_query_available,
                    user_image_available,
                    df_entries,
                ) = _set_eval(set_eval, ent_quad_df, ent_quad_image_df)

                loop_subset_df = _limit_eval_df(loop_df, eval_n)

                for qa_idx, qa_entry in tqdm(
                    loop_subset_df.iterrows(),
                    total=len(loop_subset_df),
                    desc=f"{model_name} | SET {set_eval}",
                ):
                    if curr_eval_idx < checkpoint_eval_idx:
                        curr_eval_idx += 1
                        continue

                    if session is None:
                        print(f"[INFO] Loading model for evaluation: {model_name}")
                        startup_probe = ResourceProbe().start()
                        session = ChatbotSession(ChatbotArchitecture(model_card=slm_card, is_eval_mode=False))
                        startup_report = startup_probe.stop()
                        startup_report.update({"model_name": model_name, "model_card": slm_card, "event": "model_startup"})
                        model_startup_rows.append(startup_report)
                        pd.DataFrame(model_startup_rows).to_csv(model_startup_report_path, index=False, encoding="utf-8-sig")

                    user_prompt = qa_entry["Question"]
                    user_query = qa_entry["Question"] if user_query_available else None

                    user_image = None
                    if user_image_available:
                        figure_number = json.loads(qa_entry["Figure Numbers"])[0]
                        image_path = image_records[str(figure_number)]["image_path"]
                        with Image.open(image_path) as img:
                            user_image = img.convert("RGB")

                    q_id = f"q_{qa_idx + 1:03d}"
                    answer, rag_context, resource_report = session.generate_evaluation(
                        user_prompt=user_prompt,
                        user_query=user_query,
                        user_image=user_image,
                        RAG_enabled=rag_enabled,
                        question_id=q_id,
                        model_card=slm_card,
                    )

                    evaluation_rows.append({
                        "id": q_id,
                        "model_name": model_name,
                        "architecture_config": resource_report.get("architecture_config"),
                        "section": qa_entry["Section"],
                        "subsection": qa_entry["Subsection"],
                        "question": qa_entry["Question"],
                        "reference": qa_entry["Answer"],
                        "reference_context": qa_entry["Answer Context"],
                        "response": answer,
                        "rag_context": rag_context,
                        **resource_report,
                    })

                    pd.DataFrame(evaluation_rows).to_csv(slm_checkpoint_path, index=False, encoding="utf-8-sig")
                    curr_eval_idx += 1
        finally:
            if session is not None:
                print(f"[INFO] Closing model session: {model_name}")
                session.clear_chat_history()
                session.close()
                gc.collect()
                if torch.cuda.is_available():
                    torch.cuda.empty_cache()
                    torch.cuda.ipc_collect()

    evaluation_report_df = pd.DataFrame(evaluation_rows)
    evaluation_report_df.to_csv(slm_report_path, index=False, encoding="utf-8-sig")
    print("[INFO] Saved completed SLM evaluation report:")
    print(slm_report_path)
    delete_checkpoint_if_exists(slm_checkpoint_path)

    return evaluation_report_df


if __name__ == "__main__":
    run_generation_loop()
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# evaluation_scripts/semantic_evaluation.py
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / 'semantic_evaluation.py'] = dedent(r'''
"""Local semantic evaluation module."""

from __future__ import annotations

import sys
from pathlib import Path

EVALUATION_SCRIPT_DIR = Path(__file__).resolve().parent
SCRIPT_DIR = EVALUATION_SCRIPT_DIR.parent
for _path in [SCRIPT_DIR, EVALUATION_SCRIPT_DIR]:
    if str(_path) not in sys.path:
        sys.path.insert(0, str(_path))

import gc

import numpy as np
import pandas as pd
import torch
from sentence_transformers import SentenceTransformer
from tqdm.auto import tqdm

from config import EVALUATION_MERGE_KEYS, MODEL_GENERATION_REPORT_PATH, SEMANTIC_EVALUATION_REPORT_PATH, SLM_CONFIG

SEMANTIC_EVAL_MODEL_NAME = SLM_CONFIG["TEXT_ENCODER_CARD"]

SEMANTIC_METRIC_COLS = [
    "semantic_answer_reference_similarity",
    "semantic_context_reference_similarity",
    "semantic_question_answer_alignment",
    "semantic_question_context_alignment",
]
SEMANTIC_ANSWER_METRIC_COLS = [
    "semantic_answer_reference_similarity",
    "semantic_question_answer_alignment",
]
SEMANTIC_CONTEXT_METRIC_COLS = [
    "semantic_context_reference_similarity",
    "semantic_question_context_alignment",
]


def _is_missing_text(text) -> bool:
    return bool(text is None) or bool(pd.isna(text)) or bool(str(text).strip() == "")


def run_semantic_evaluation(evaluation_report_df: pd.DataFrame | None = None, force_regenerate: bool = False) -> pd.DataFrame:
    semantic_eval_path = SEMANTIC_EVALUATION_REPORT_PATH

    if semantic_eval_path.exists() and not force_regenerate:
        print("[INFO] Completed local semantic evaluation found.")
        print(f"[INFO] Loading semantic evaluation from: {semantic_eval_path}")
        return pd.read_csv(semantic_eval_path, encoding="utf-8-sig")

    if evaluation_report_df is None:
        evaluation_report_df = pd.read_csv(MODEL_GENERATION_REPORT_PATH, encoding="utf-8-sig")

    print("[INFO] Loading Semantic Evaluation Encoder")
    semantic_eval_encoder = SentenceTransformer(
        SEMANTIC_EVAL_MODEL_NAME,
        device="cuda" if torch.cuda.is_available() else "cpu",
    )

    semantic_embedding_cache = {}

    def _embed_semantic_text(text):
        text = str(text).strip()
        if text not in semantic_embedding_cache:
            semantic_embedding_cache[text] = semantic_eval_encoder.encode(
                text,
                normalize_embeddings=True,
                convert_to_numpy=True,
            )
        return semantic_embedding_cache[text]

    def semantic_cosine(text_a: str, text_b: str):
        if _is_missing_text(text_a) or _is_missing_text(text_b):
            return None
        return float(np.dot(_embed_semantic_text(text_a), _embed_semantic_text(text_b)))

    def max_semantic_cosine(text: str, rag_context: str):
        if _is_missing_text(text) or _is_missing_text(rag_context):
            return None
        rag_passages = [passage.strip() for passage in str(rag_context).split("\n\n") if passage.strip()]
        if len(rag_passages) == 0:
            return None
        return max(semantic_cosine(text, rag_passage) for rag_passage in rag_passages)

    semantic_eval_rows = []
    for _, row in tqdm(evaluation_report_df.iterrows(), total=len(evaluation_report_df), desc="Local semantic evaluation"):
        semantic_eval_rows.append({
            "id": row["id"],
            "model_name": row["model_name"],
            "architecture_config": row["architecture_config"],
            "semantic_answer_reference_similarity": semantic_cosine(row["response"], row["reference"]),
            "semantic_context_reference_similarity": max_semantic_cosine(row["reference_context"], row["rag_context"]),
            "semantic_question_answer_alignment": semantic_cosine(row["question"], row["response"]),
            "semantic_question_context_alignment": max_semantic_cosine(row["question"], row["rag_context"]),
        })

    del semantic_eval_encoder
    semantic_embedding_cache.clear()
    gc.collect()
    if torch.cuda.is_available():
        torch.cuda.empty_cache()
        torch.cuda.ipc_collect()

    semantic_eval_df = pd.DataFrame(semantic_eval_rows)
    semantic_eval_df["semantic_answer_mean"] = semantic_eval_df[SEMANTIC_ANSWER_METRIC_COLS].mean(axis=1)
    semantic_eval_df["semantic_context_mean"] = semantic_eval_df[SEMANTIC_CONTEXT_METRIC_COLS].mean(axis=1)
    semantic_eval_df["semantic_mean"] = semantic_eval_df[SEMANTIC_METRIC_COLS].mean(axis=1)

    semantic_eval_df.to_csv(semantic_eval_path, index=False, encoding="utf-8-sig")
    print("[INFO] Saved local semantic evaluation:")
    print(semantic_eval_path)
    return semantic_eval_df


if __name__ == "__main__":
    run_semantic_evaluation()
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# evaluation_scripts/ragas_evaluation.py
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / 'ragas_evaluation.py'] = dedent(r'''
"""RAGAS LLM-as-judge evaluation module.

This module follows the notebook's four-worker checkpointing design, but writes
all outputs under the persistent project-root evaluation directory.
"""

from __future__ import annotations

import sys
from pathlib import Path

EVALUATION_SCRIPT_DIR = Path(__file__).resolve().parent
SCRIPT_DIR = EVALUATION_SCRIPT_DIR.parent
for _path in [SCRIPT_DIR, EVALUATION_SCRIPT_DIR]:
    if str(_path) not in sys.path:
        sys.path.insert(0, str(_path))

import asyncio
from datetime import datetime

import pandas as pd
from openai import AsyncOpenAI
from ragas.embeddings import OpenAIEmbeddings
from ragas.llms import llm_factory
from ragas.metrics.collections import (
    AnswerCorrectness,
    AnswerRelevancy,
    ContextPrecision,
    ContextRecall,
    Faithfulness,
)
from tqdm.auto import tqdm

from config import (
    ARCHITECTURE_ORDER,
    EVALUATION_CHECKPOINTS_DIR,
    EVALUATION_MERGE_KEYS,
    FINAL_EVALUATION_REPORT_PATH,
    MODEL_ORDER,
    OPENAI_API_KEY,
    RAGAS_EMBEDDING_MODEL,
    RAGAS_EVALUATION_MODE,
    RAGAS_EVALUATION_REPORT_PATH,
    RAGAS_EVALUATOR_MODEL,
    RAGAS_MAX_RETRIES,
    RAGAS_N_WORKERS,
    RAGAS_RETRY_BASE_SECONDS,
)
from evaluation_scripts.checkpointing import delete_checkpoint_if_exists

RAGAS_METRIC_COLS = ["faithfulness", "answer_relevancy", "answer_correctness", "context_precision", "context_recall"]
RAGAS_ANSWER_METRIC_COLS = ["answer_relevancy", "answer_correctness"]
RAGAS_CONTEXT_METRIC_COLS = ["faithfulness", "context_precision", "context_recall"]


def _rag_context_to_list(text):
    if text is None:
        return []
    try:
        if pd.isna(text):
            return []
    except (TypeError, ValueError):
        pass
    text = str(text).strip()
    if text == "" or text.lower() == "nan" or text.lower() == "insufficient context!":
        return []
    return [context.strip() for context in text.split("\n\n") if context.strip()]


def _mean_available(values):
    clean_values = []
    for value in values:
        if value is None:
            continue
        try:
            if pd.isna(value):
                continue
        except (TypeError, ValueError):
            pass
        clean_values.append(float(value))
    if len(clean_values) == 0:
        return None
    return float(sum(clean_values) / len(clean_values))


def _add_ragas_means(row_dict):
    row_dict["ragas_answer_mean"] = _mean_available([row_dict.get("answer_relevancy"), row_dict.get("answer_correctness")])
    row_dict["ragas_context_mean"] = _mean_available([row_dict.get("faithfulness"), row_dict.get("context_precision"), row_dict.get("context_recall")])
    row_dict["ragas_mean"] = _mean_available([
        row_dict.get("faithfulness"),
        row_dict.get("answer_relevancy"),
        row_dict.get("answer_correctness"),
        row_dict.get("context_precision"),
        row_dict.get("context_recall"),
    ])
    return row_dict


def _row_key(row):
    return (str(row["id"]), str(row["model_name"]), str(row["architecture_config"]))


def _has_retrieved_contexts(row):
    contexts = row["retrieved_contexts"]
    return isinstance(contexts, list) and len(contexts) > 0


def _question_sort_value(id_value):
    try:
        return int(str(id_value).replace("q_", ""))
    except Exception:
        return 10**9


def _ragas_eval_columns():
    return EVALUATION_MERGE_KEYS + RAGAS_METRIC_COLS + ["ragas_answer_mean", "ragas_context_mean", "ragas_mean"]


def _sort_ragas_eval_df(df):
    columns = _ragas_eval_columns()
    df = df.copy()
    if df.empty:
        return df.reindex(columns=columns)
    for col in columns:
        if col not in df.columns:
            df[col] = None
    df = df[columns].copy()
    df["_model_order"] = df["model_name"].map({name: idx for idx, name in enumerate(MODEL_ORDER)})
    df["_architecture_order"] = df["architecture_config"].map({name: idx for idx, name in enumerate(ARCHITECTURE_ORDER)})
    df["_id_order"] = df["id"].map(_question_sort_value)
    return df.sort_values(by=["_model_order", "_architecture_order", "_id_order", "id"], kind="mergesort").drop(columns=["_model_order", "_architecture_order", "_id_order"]).reset_index(drop=True)


def _deduplicate_ragas_eval_df(df):
    if df.empty:
        return df.reindex(columns=_ragas_eval_columns())
    return _sort_ragas_eval_df(df.drop_duplicates(subset=EVALUATION_MERGE_KEYS, keep="last"))


def _load_checkpoint_records(path):
    if not path.exists():
        return []
    df = pd.read_csv(path, encoding="utf-8-sig")
    if df.empty:
        return []
    return df.to_dict("records")


def _save_checkpoint_records(path, records):
    checkpoint_df = pd.DataFrame(records)
    for col in _ragas_eval_columns():
        if col not in checkpoint_df.columns:
            checkpoint_df[col] = None
    checkpoint_df = _deduplicate_ragas_eval_df(checkpoint_df)
    checkpoint_df.to_csv(path, index=False, encoding="utf-8-sig")


def _ordered_group(df, model_name, architecture_config):
    group = df[(df["model_name"] == model_name) & (df["architecture_config"] == architecture_config)].copy()
    group["_id_order"] = group["id"].map(_question_sort_value)
    return group.sort_values(["_id_order", "id"], kind="mergesort").drop(columns=["_id_order"]).reset_index(drop=True)


def _slice_group(df, model_name, architecture_config, start, end):
    return _ordered_group(df, model_name, architecture_config).iloc[start - 1:end].copy()


def _all_group(df, model_name, architecture_config):
    return _ordered_group(df, model_name, architecture_config)


def _build_ragas_worker_splits(df):
    if len(MODEL_ORDER) != 2:
        raise ValueError("The four-worker split expects exactly two SLM model cards.")
    slm_1, slm_2 = MODEL_ORDER
    worker_splits = {
        1: [_slice_group(df, slm_1, "SLM", 1, 125), _slice_group(df, slm_1, "Text-RAG", 1, 125), _all_group(df, slm_1, "Image-Text-RAG")],
        2: [_slice_group(df, slm_1, "SLM", 126, 250), _slice_group(df, slm_1, "Text-RAG", 126, 250), _all_group(df, slm_1, "Image-RAG")],
        3: [_slice_group(df, slm_2, "SLM", 1, 125), _slice_group(df, slm_2, "Text-RAG", 1, 125), _all_group(df, slm_2, "Image-Text-RAG")],
        4: [_slice_group(df, slm_2, "SLM", 126, 250), _slice_group(df, slm_2, "Text-RAG", 126, 250), _all_group(df, slm_2, "Image-RAG")],
    }
    return {worker_id: pd.concat(parts, ignore_index=True) for worker_id, parts in worker_splits.items()}


def _validate_worker_splits(worker_dfs, source_df):
    expected_keys = {_row_key(row) for _, row in source_df.iterrows()}
    observed_keys = []
    for worker_id, worker_df in worker_dfs.items():
        worker_keys = [_row_key(row) for _, row in worker_df.iterrows()]
        if len(worker_keys) - len(set(worker_keys)) > 0:
            raise ValueError(f"Worker {worker_id} has duplicate row keys.")
        observed_keys.extend(worker_keys)
    if expected_keys != set(observed_keys) or len(observed_keys) != len(set(observed_keys)):
        raise ValueError("Invalid RAGAS worker split.")
    return pd.DataFrame([
        {
            "worker_id": worker_id,
            "rows": len(worker_df),
            "slm_rows": int((worker_df["architecture_config"] == "SLM").sum()),
            "rag_rows": int((worker_df["architecture_config"] != "SLM").sum()),
            "estimated_metric_calls": int((worker_df["architecture_config"] == "SLM").sum()) * 2 + int((worker_df["architecture_config"] != "SLM").sum()) * 5,
        }
        for worker_id, worker_df in worker_dfs.items()
    ])


def _prepare_ragas_df(evaluation_report_df):
    ragas_df = evaluation_report_df[EVALUATION_MERGE_KEYS + ["section", "subsection", "question", "response", "rag_context", "reference", "reference_context"]].copy()
    ragas_df["user_input"] = ragas_df["question"]
    ragas_df["retrieved_contexts"] = ragas_df["rag_context"].map(_rag_context_to_list)
    return ragas_df[EVALUATION_MERGE_KEYS + ["section", "subsection", "user_input", "response", "retrieved_contexts", "reference", "reference_context"]].copy()


def run_ragas_evaluation(evaluation_report_df: pd.DataFrame | None = None, mode: str | None = None) -> pd.DataFrame:
    mode = mode or RAGAS_EVALUATION_MODE
    ragas_eval_path = RAGAS_EVALUATION_REPORT_PATH
    worker_paths = {worker_id: EVALUATION_CHECKPOINTS_DIR / f"ragas_worker_{worker_id}_checkpoint.csv" for worker_id in range(1, RAGAS_N_WORKERS + 1)}
    failure_log_path = EVALUATION_CHECKPOINTS_DIR / "ragas_worker_failure_log.csv"

    def load_all_worker_checkpoint_records():
        records = []
        for worker_id, path in worker_paths.items():
            worker_records = _load_checkpoint_records(path)
            if worker_records:
                print(f"[INFO] Loaded worker {worker_id} checkpoint rows: {len(worker_records)}")
            records.extend(worker_records)
        return records

    def delete_worker_checkpoints_if_exists():
        for path in worker_paths.values():
            delete_checkpoint_if_exists(path)
        delete_checkpoint_if_exists(failure_log_path)

    def append_failure_log(failure_records):
        if failure_records:
            pd.DataFrame(failure_records).to_csv(failure_log_path, index=False, encoding="utf-8-sig")

    if ragas_eval_path.exists() and mode != "force_rerun":
        print("[INFO] Completed RAGAS evaluation file found. Skipping OpenAI evaluator calls.")
        return _deduplicate_ragas_eval_df(pd.read_csv(ragas_eval_path, encoding="utf-8-sig"))

    if mode == "reuse_only":
        print("[INFO] RAGAS evaluation is disabled. Loading worker checkpoints only, if present.")
        return _deduplicate_ragas_eval_df(pd.DataFrame(load_all_worker_checkpoint_records()))

    if mode not in ["run_if_missing", "force_rerun"]:
        raise ValueError("Invalid RAGAS evaluation mode. Use reuse_only, run_if_missing, or force_rerun.")

    if not bool(OPENAI_API_KEY):
        raise RuntimeError("OPENAI_API_KEY is missing. Set OPENAI_API_KEY in .secrets or use mode='reuse_only'.")

    if evaluation_report_df is None:
        evaluation_report_df = pd.read_csv(FINAL_EVALUATION_REPORT_PATH if FINAL_EVALUATION_REPORT_PATH.exists() else FINAL_EVALUATION_REPORT_PATH.parent / "model_gen_report.csv", encoding="utf-8-sig")

    ragas_df = _prepare_ragas_df(evaluation_report_df)

    if mode == "force_rerun":
        print("[WARN] force_rerun enabled. Existing RAGAS outputs will be ignored.")
        delete_checkpoint_if_exists(ragas_eval_path)
        delete_worker_checkpoints_if_exists()
    else:
        delete_checkpoint_if_exists(failure_log_path)

    worker_dfs = _build_ragas_worker_splits(ragas_df)
    worker_summary_df = _validate_worker_splits(worker_dfs, ragas_df)
    print("[INFO] Four-worker RAGAS split summary:")
    print(worker_summary_df)

    worker_checkpoint_records = {worker_id: _load_checkpoint_records(path) for worker_id, path in worker_paths.items()}
    completed_keys = {_row_key(row) for records in worker_checkpoint_records.values() for row in records}

    openai_client = AsyncOpenAI(api_key=OPENAI_API_KEY)
    ragas_llm = llm_factory(RAGAS_EVALUATOR_MODEL, client=openai_client)
    if hasattr(ragas_llm, "model_args"):
        ragas_llm.model_args.pop("max_tokens", None)
        ragas_llm.model_args.pop("top_p", None)
        ragas_llm.model_args["max_completion_tokens"] = 8192
        ragas_llm.model_args["temperature"] = 1

    ragas_embeddings = OpenAIEmbeddings(client=openai_client, model=RAGAS_EMBEDDING_MODEL)
    faithfulness = Faithfulness(llm=ragas_llm)
    answer_relevancy = AnswerRelevancy(llm=ragas_llm, embeddings=ragas_embeddings)
    answer_correctness = AnswerCorrectness(llm=ragas_llm, embeddings=ragas_embeddings)
    context_precision = ContextPrecision(llm=ragas_llm)
    context_recall = ContextRecall(llm=ragas_llm)

    async def score_ragas_row(row):
        scores = {metric: None for metric in RAGAS_METRIC_COLS}
        scores["answer_relevancy"] = (await answer_relevancy.ascore(user_input=row["user_input"], response=row["response"])).value
        scores["answer_correctness"] = (await answer_correctness.ascore(user_input=row["user_input"], response=row["response"], reference=row["reference"])).value
        if _has_retrieved_contexts(row):
            scores["faithfulness"] = (await faithfulness.ascore(user_input=row["user_input"], response=row["response"], retrieved_contexts=row["retrieved_contexts"])).value
            scores["context_precision"] = (await context_precision.ascore(user_input=row["user_input"], reference=row["reference"], retrieved_contexts=row["retrieved_contexts"])).value
            scores["context_recall"] = (await context_recall.ascore(user_input=row["user_input"], reference=row["reference"], retrieved_contexts=row["retrieved_contexts"])).value
        return scores

    async def score_ragas_row_with_retries(row, worker_id):
        last_error = None
        for attempt in range(1, RAGAS_MAX_RETRIES + 1):
            try:
                return await score_ragas_row(row)
            except Exception as exc:
                last_error = exc
                if attempt >= RAGAS_MAX_RETRIES:
                    break
                sleep_seconds = RAGAS_RETRY_BASE_SECONDS * attempt
                print(f"[WARN] worker={worker_id} key={_row_key(row)} attempt={attempt}/{RAGAS_MAX_RETRIES} failed: {exc}. Retrying in {sleep_seconds:.1f}s.")
                await asyncio.sleep(sleep_seconds)
        raise last_error

    async def ragas_worker(worker_id, worker_df, pbar, checkpoint_write_lock, failure_write_lock, failure_records):
        worker_path = worker_paths[worker_id]
        worker_rows = worker_checkpoint_records.get(worker_id, [])
        worker_completed_keys = {_row_key(row) for row in worker_rows}
        for _, row in worker_df.iterrows():
            current_key = _row_key(row)
            if current_key in completed_keys or current_key in worker_completed_keys:
                continue
            try:
                scores = await score_ragas_row_with_retries(row, worker_id)
                row_dict = {"id": row["id"], "model_name": row["model_name"], "architecture_config": row["architecture_config"], **scores}
                row_dict = _add_ragas_means(row_dict)
                worker_rows.append(row_dict)
                worker_completed_keys.add(current_key)
                completed_keys.add(current_key)
                async with checkpoint_write_lock:
                    _save_checkpoint_records(worker_path, worker_rows)
                    pbar.update(1)
            except Exception as exc:
                failure_row = {
                    "timestamp": datetime.now().isoformat(timespec="seconds"),
                    "worker_id": worker_id,
                    "id": row["id"],
                    "model_name": row["model_name"],
                    "architecture_config": row["architecture_config"],
                    "error_type": type(exc).__name__,
                    "error_message": str(exc),
                }
                async with failure_write_lock:
                    failure_records.append(failure_row)
                    append_failure_log(failure_records)
                    pbar.update(1)
                print(f"[ERROR] worker={worker_id} key={current_key}: {type(exc).__name__}: {exc}")

    async def run_ragas_four_worker_evaluation():
        checkpoint_write_lock = asyncio.Lock()
        failure_write_lock = asyncio.Lock()
        failure_records = []
        total_remaining = sum(1 for worker_df in worker_dfs.values() for _, row in worker_df.iterrows() if _row_key(row) not in completed_keys)
        with tqdm(total=total_remaining, desc="RAGAS 4-worker") as pbar:
            await asyncio.gather(*[
                asyncio.create_task(ragas_worker(worker_id, worker_df, pbar, checkpoint_write_lock, failure_write_lock, failure_records))
                for worker_id, worker_df in worker_dfs.items()
            ])
        merged_records = load_all_worker_checkpoint_records()
        merged_df = _deduplicate_ragas_eval_df(pd.DataFrame(merged_records))
        observed_keys = {_row_key(row) for _, row in merged_df.iterrows()}
        expected_keys = {_row_key(row) for _, row in ragas_df.iterrows()}
        missing_keys = expected_keys - observed_keys
        if failure_records or missing_keys:
            print("[ERROR] RAGAS four-worker evaluation did not complete cleanly.")
            print("[ERROR] Successful rows:", len(observed_keys))
            print("[ERROR] Expected rows:", len(expected_keys))
            print("[ERROR] Missing rows:", len(missing_keys))
            print("[ERROR] Failure rows:", len(failure_records))
            raise RuntimeError("RAGAS evaluation is incomplete. Worker checkpoints are preserved for resumption.")
        merged_df.to_csv(ragas_eval_path, index=False, encoding="utf-8-sig")
        delete_checkpoint_if_exists(failure_log_path)
        print("[INFO] Saved completed merged RAGAS evaluation:")
        print(ragas_eval_path)
        return merged_df

    return asyncio.run(run_ragas_four_worker_evaluation())


if __name__ == "__main__":
    run_ragas_evaluation()
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# evaluation_scripts/final_report.py
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / 'final_report.py'] = dedent(r'''
"""Final evaluation report assembly."""

from __future__ import annotations

import sys
from pathlib import Path

EVALUATION_SCRIPT_DIR = Path(__file__).resolve().parent
SCRIPT_DIR = EVALUATION_SCRIPT_DIR.parent
for _path in [SCRIPT_DIR, EVALUATION_SCRIPT_DIR]:
    if str(_path) not in sys.path:
        sys.path.insert(0, str(_path))

import pandas as pd

from config import (
    ARCHITECTURE_ORDER,
    EVALUATION_MERGE_KEYS,
    FINAL_EVALUATION_REPORT_PATH,
    MODEL_GENERATION_REPORT_PATH,
    MODEL_ORDER,
    RAGAS_EVALUATION_REPORT_PATH,
    SEMANTIC_EVALUATION_REPORT_PATH,
)
from ragas_evaluation import RAGAS_METRIC_COLS
from semantic_evaluation import SEMANTIC_METRIC_COLS

SEMANTIC_SUMMARY_COLS = ["semantic_answer_mean", "semantic_context_mean", "semantic_mean"]
RAGAS_SUMMARY_COLS = ["ragas_answer_mean", "ragas_context_mean", "ragas_mean"]
MODULE_METRIC_COLS = SEMANTIC_METRIC_COLS + SEMANTIC_SUMMARY_COLS + RAGAS_METRIC_COLS + RAGAS_SUMMARY_COLS


def assemble_final_report(
    generation_df: pd.DataFrame | None = None,
    semantic_eval_df: pd.DataFrame | None = None,
    ragas_eval_df: pd.DataFrame | None = None,
) -> pd.DataFrame:
    if generation_df is None:
        generation_df = pd.read_csv(MODEL_GENERATION_REPORT_PATH, encoding="utf-8-sig")
    if semantic_eval_df is None:
        semantic_eval_df = pd.read_csv(SEMANTIC_EVALUATION_REPORT_PATH, encoding="utf-8-sig")
    if ragas_eval_df is None:
        ragas_eval_df = pd.read_csv(RAGAS_EVALUATION_REPORT_PATH, encoding="utf-8-sig")

    evaluation_report_df = generation_df.drop(
        columns=[column for column in MODULE_METRIC_COLS if column in generation_df.columns],
        errors="ignore",
    )

    evaluation_report_df = evaluation_report_df.merge(semantic_eval_df, on=EVALUATION_MERGE_KEYS, how="left")
    evaluation_report_df = evaluation_report_df.merge(ragas_eval_df, on=EVALUATION_MERGE_KEYS, how="left")

    evaluation_report_df["_model_order"] = evaluation_report_df["model_name"].map({name: idx for idx, name in enumerate(MODEL_ORDER)})
    evaluation_report_df["_architecture_order"] = evaluation_report_df["architecture_config"].map({name: idx for idx, name in enumerate(ARCHITECTURE_ORDER)})
    evaluation_report_df["_question_order"] = evaluation_report_df["id"].astype(str).str.replace("q_", "", regex=False).astype(int)

    evaluation_report_df = evaluation_report_df.sort_values(
        by=["_model_order", "_architecture_order", "_question_order"],
        kind="stable",
    ).drop(columns=["_model_order", "_architecture_order", "_question_order"])

    evaluation_report_df.to_csv(FINAL_EVALUATION_REPORT_PATH, index=False, encoding="utf-8-sig")
    print("[INFO] Saved final evaluation report:")
    print(FINAL_EVALUATION_REPORT_PATH)
    return evaluation_report_df


if __name__ == "__main__":
    assemble_final_report()
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# evaluation_scripts/reporting.py
FILES[RUN_TIME_EVALUATION_SCRIPT_DIR / 'reporting.py'] = dedent(r'''
"""Reporting helper for figures and tables.

This module intentionally writes figures/tables to the persistent evaluation
folder. It does not write anything under run_time.
"""

from __future__ import annotations

import sys
from pathlib import Path

EVALUATION_SCRIPT_DIR = Path(__file__).resolve().parent
SCRIPT_DIR = EVALUATION_SCRIPT_DIR.parent
for _path in [SCRIPT_DIR, EVALUATION_SCRIPT_DIR]:
    if str(_path) not in sys.path:
        sys.path.insert(0, str(_path))

import re

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from config import (
    ARCHITECTURE_ORDER,
    EVALUATION_FIGURES_DIR,
    EVALUATION_REPORTS_DIR,
    EVALUATION_TABLES_DIR,
    FINAL_EVALUATION_REPORT_PATH,
    FINAL_GPT_4O_MINI_EVALUATION_REPORT_PATH,
    MODEL_GENERATION_REPORT_PATH,
    MODEL_ORDER,
    MODEL_STARTUP_REPORT_PATH,
    RAGAS_EVALUATION_REPORT_PATH,
    RAGAS_GPT_4O_MINI_REPORT_PATH,
    SEMANTIC_EVALUATION_REPORT_PATH,
)

RESPONSE_PATTERN_RULES = {
    "missing_context": re.compile(r"no reference passage|no context|context was not provided|no retrieved context|passage is missing|insufficient context", re.I),
    "grounding_exposure": re.compile(r"retrieved passage|retrieved context|reference passage|retrieval result|retrieval process|rag system|\brag\b|faiss|source-grounded|not source-grounded|grounded response|reference-supported|response mode", re.I),
    "refusal": re.compile(r"i cannot answer|i can't answer|i am unable to answer|unable to provide an answer|cannot provide an answer", re.I),
}


def classify_response_pattern(text: str) -> str:
    text = "" if pd.isna(text) else str(text)
    if RESPONSE_PATTERN_RULES["missing_context"].search(text):
        return "Missing context statement"
    if RESPONSE_PATTERN_RULES["grounding_exposure"].search(text):
        return "Grounding/RAG exposure"
    if RESPONSE_PATTERN_RULES["refusal"].search(text):
        return "Refusal"
    return "Faultless"


def _mean_sd(series, scale: float = 100.0) -> str:
    series = pd.to_numeric(series, errors="coerce").dropna() * scale
    if series.empty:
        return "—"
    return f"{series.mean():.2f} ± {series.std(ddof=1):.2f}"


def build_reporting_outputs() -> dict[str, pd.DataFrame]:
    EVALUATION_FIGURES_DIR.mkdir(parents=True, exist_ok=True)
    EVALUATION_TABLES_DIR.mkdir(parents=True, exist_ok=True)

    outputs: dict[str, pd.DataFrame] = {}

    evaluation_report_df = pd.read_csv(FINAL_EVALUATION_REPORT_PATH, encoding="utf-8-sig")

    # Response pattern table and figure.
    response_pattern_df = evaluation_report_df.copy()
    response_pattern_df["response_pattern"] = response_pattern_df["response"].map(classify_response_pattern)
    pattern_table = (
        response_pattern_df.groupby(["model_name", "architecture_config", "response_pattern"], dropna=False)
        .size()
        .reset_index(name="count")
    )
    pattern_table.to_csv(EVALUATION_TABLES_DIR / "table_4_1_response_pattern_count.csv", index=False, encoding="utf-8-sig")
    outputs["response_pattern_table"] = pattern_table

    fig_df = response_pattern_df.groupby(["model_name", "response_pattern"]).size().reset_index(name="count")
    pivot = fig_df.pivot(index="model_name", columns="response_pattern", values="count").fillna(0)
    pivot = pivot.reindex(MODEL_ORDER)
    ax = pivot.plot(kind="bar", figsize=(9, 5))
    ax.set_xlabel("Model card")
    ax.set_ylabel("Response count")
    ax.set_title("Response Pattern")
    plt.tight_layout()
    plt.savefig(EVALUATION_FIGURES_DIR / "figure_4_2_response_pattern_count.png", dpi=300)
    plt.close()

    # Compact semantic summary.
    semantic_cols = [
        "semantic_answer_mean",
        "semantic_answer_reference_similarity",
        "semantic_question_answer_alignment",
    ]
    semantic_summary = []
    for (model_name, arch), group in evaluation_report_df.groupby(["model_name", "architecture_config"], sort=False):
        row = {"model_name": model_name, "architecture_config": arch, "n": len(group)}
        for col in semantic_cols:
            if col in group.columns:
                row[col] = _mean_sd(group[col])
        semantic_summary.append(row)
    semantic_summary_df = pd.DataFrame(semantic_summary)
    semantic_summary_df.to_csv(EVALUATION_TABLES_DIR / "semantic_answer_summary_runtime.csv", index=False, encoding="utf-8-sig")
    outputs["semantic_summary"] = semantic_summary_df

    # Compact RAGAS summary.
    ragas_cols = ["ragas_answer_mean", "answer_relevancy", "answer_correctness", "ragas_context_mean", "ragas_mean"]
    ragas_summary = []
    for (model_name, arch), group in evaluation_report_df.groupby(["model_name", "architecture_config"], sort=False):
        row = {"model_name": model_name, "architecture_config": arch, "n": len(group)}
        for col in ragas_cols:
            if col in group.columns:
                row[col] = _mean_sd(group[col])
        ragas_summary.append(row)
    ragas_summary_df = pd.DataFrame(ragas_summary)
    ragas_summary_df.to_csv(EVALUATION_TABLES_DIR / "ragas_summary_runtime.csv", index=False, encoding="utf-8-sig")
    outputs["ragas_summary"] = ragas_summary_df

    print("[INFO] Reporting outputs written to:")
    print(EVALUATION_FIGURES_DIR)
    print(EVALUATION_TABLES_DIR)
    return outputs


if __name__ == "__main__":
    build_reporting_outputs()
''').lstrip("\n").rstrip() + "\n"

# -----------------------------------------------------------------------------
# runtime_pipeline.py
FILES[RUN_TIME_SCRIPT_DIR / 'runtime_pipeline.py'] = dedent(r'''
"""Command-line orchestration for the run-time ENT RAG evaluation pipeline."""

from __future__ import annotations

import sys
from pathlib import Path

SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
    sys.path.insert(0, str(SCRIPT_DIR))

import argparse

from evaluation_scripts.final_report import assemble_final_report
from evaluation_scripts.generation_loop import run_generation_loop
from evaluation_scripts.ragas_evaluation import run_ragas_evaluation
from evaluation_scripts.reporting import build_reporting_outputs
from evaluation_scripts.semantic_evaluation import run_semantic_evaluation


def main() -> None:
    parser = argparse.ArgumentParser(description="Run the ENT RAG run-time evaluation pipeline.")
    parser.add_argument("--generation", action="store_true", help="Run or reuse the SLM generation loop.")
    parser.add_argument("--semantic", action="store_true", help="Run or reuse local semantic evaluation.")
    parser.add_argument("--ragas", action="store_true", help="Run or reuse RAGAS evaluation.")
    parser.add_argument("--final-report", action="store_true", help="Assemble the final merged evaluation report.")
    parser.add_argument("--reporting", action="store_true", help="Build reporting outputs from completed reports.")
    parser.add_argument("--all", action="store_true", help="Run all stages in notebook order.")
    parser.add_argument("--force-generation", action="store_true", help="Force regeneration of the SLM-loop report.")
    parser.add_argument("--force-semantic", action="store_true", help="Force regeneration of semantic evaluation.")
    parser.add_argument("--ragas-mode", default=None, choices=[None, "reuse_only", "run_if_missing", "force_rerun"], help="Override RAGAS evaluation mode.")
    args = parser.parse_args()

    run_all = args.all or not any([args.generation, args.semantic, args.ragas, args.final_report, args.reporting])

    generation_df = None
    semantic_df = None
    ragas_df = None

    if run_all or args.generation:
        generation_df = run_generation_loop(force_regenerate=args.force_generation)
    if run_all or args.semantic:
        semantic_df = run_semantic_evaluation(generation_df, force_regenerate=args.force_semantic)
    if run_all or args.ragas:
        ragas_df = run_ragas_evaluation(generation_df, mode=args.ragas_mode)
    if run_all or args.final_report:
        assemble_final_report(generation_df, semantic_df, ragas_df)
    if run_all or args.reporting:
        build_reporting_outputs()


if __name__ == "__main__":
    main()
''').lstrip("\n").rstrip() + "\n"


def require_secrets_file() -> None:
    """Validate that the project-level .secrets file exists and contains required keys.

    The function returns None on success. It raises a specific exception when
    the secrets file is missing, malformed, unreadable, or incomplete.
    """

    if not SECRETS_PATH.exists():
        raise FileNotFoundError(
            "\n[ERROR] Required secrets file was not found.\n"
            f"Expected path: {SECRETS_PATH}\n\n"
            "Create a .secrets file in the project root before running the runtime bootstrap.\n"
            "Expected format:\n"
            "HF_TOKEN=your_huggingface_token_here\n"
            "OPENAI_API_KEY=your_openai_api_key_here\n"
        )

    if not SECRETS_PATH.is_file():
        raise IsADirectoryError(
            "\n[ERROR] The .secrets path exists but is not a file.\n"
            f"Path: {SECRETS_PATH}\n"
        )

    try:
        raw_text = SECRETS_PATH.read_text(encoding="utf-8")
    except PermissionError as exc:
        raise PermissionError(
            "\n[ERROR] The .secrets file exists but cannot be read because of permission restrictions.\n"
            f"Path: {SECRETS_PATH}\n"
        ) from exc

    if raw_text.strip() == "":
        raise ValueError(
            "\n[ERROR] The .secrets file is empty.\n"
            f"Path: {SECRETS_PATH}\n\n"
            "Expected format:\n"
            "HF_TOKEN=your_huggingface_token_here\n"
            "OPENAI_API_KEY=your_openai_api_key_here\n"
        )

    parsed_secrets = {}

    for line_number, line in enumerate(raw_text.splitlines(), start=1):
        stripped_line = line.strip()

        if stripped_line == "" or stripped_line.startswith("#"):
            continue

        if stripped_line.startswith("export "):
            stripped_line = stripped_line.removeprefix("export ").strip()

        if "=" not in stripped_line:
            raise ValueError(
                "\n[ERROR] Malformed line in .secrets file.\n"
                f"Path: {SECRETS_PATH}\n"
                f"Line {line_number}: {line}\n\n"
                "Each non-empty, non-comment line must use KEY=value format."
            )

        key, value = stripped_line.split("=", 1)
        key = key.strip()
        value = value.strip().strip('"').strip("'")

        if key == "":
            raise ValueError(
                "\n[ERROR] Empty key found in .secrets file.\n"
                f"Path: {SECRETS_PATH}\n"
                f"Line {line_number}: {line}\n"
            )

        parsed_secrets[key] = value

    missing_keys = [key for key in REQUIRED_SECRET_KEYS if key not in parsed_secrets]

    if missing_keys:
        raise KeyError(
            "\n[ERROR] Missing required key(s) in .secrets file.\n"
            f"Path: {SECRETS_PATH}\n"
            f"Missing: {', '.join(missing_keys)}\n\n"
            "Expected keys:\n"
            "HF_TOKEN=your_huggingface_token_here\n"
            "OPENAI_API_KEY=your_openai_api_key_here\n"
        )

    blank_keys = [key for key in REQUIRED_SECRET_KEYS if parsed_secrets.get(key, "").strip() == ""]

    if blank_keys:
        raise ValueError(
            "\n[ERROR] Required key(s) in .secrets file have blank values.\n"
            f"Path: {SECRETS_PATH}\n"
            f"Blank: {', '.join(blank_keys)}\n"
        )

    print("[INFO] secrets: .secrets found and required keys are populated.")
    return None

def write_file(path: Path, content: str) -> None:
    if path.exists() and not OVERWRITE_MODULES:
        print(f"[INFO][SKIP] file exists: {path.relative_to(PROJECT_ROOT)}")
        return

    existed_before = path.exists()
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(content, encoding="utf-8")
    action = "OVERWRITE" if existed_before else "CREATE"
    print(f"[INFO][{action}] file: {path.relative_to(PROJECT_ROOT)}")


def main() -> None:
    print(f"Project root: {PROJECT_ROOT}")

    require_secrets_file()

    if RESET_RUN_TIME_DIR and RUN_TIME_DIR.exists():
        print("[INFO] Resetting .../run_time. Evaluation outputs are outside run_time and will be preserved.")
        shutil.rmtree(RUN_TIME_DIR)

    if OVERWRITE_MODULES and not RESET_RUN_TIME_DIR:
        print("[INFO] Existing run-time modules are overwritten. You can disable this by setting OVERWRITE_MODULES = False.")

    for directory in DIRECTORIES:
        directory.mkdir(parents=True, exist_ok=True)
        print(f"[INFO] directory: {directory.relative_to(PROJECT_ROOT)}")

    print("=" * 100)

    for path, content in FILES.items():
        write_file(path, content)

    print("=" * 100)

    if RUN_RUNTIME_PIPELINE_AFTER_BOOTSTRAP:
        subprocess.run(
            [sys.executable, str(RUN_TIME_SCRIPT_EXEC_PY_PATH)],
            cwd=PROJECT_ROOT,
            check=True,
        )

    print("\nRun-time implementation files are populated.")
    print("[INFO] Evaluation outputs are preserved outside run_time.")


if __name__ == "__main__":
    main()