worker.py

import asyncio
import logging
import os
import shutil
import httpx
import aiofiles
import json
import zipfile
import io
import subprocess
import socket
import docker
from typing import List, Dict, Any, Optional
from models import Job
from queue_manager import job_queue
from isolate_runner import IsolateRunner
from llm_judge import llm_judge_case, get_llm_judge

logger = logging.getLogger(__name__)

class ProxyManager:
    """管理 TinyProxy 實例"""
    def __init__(self, worker_id: int):
        self.worker_id = worker_id
        self.port = 8888 + worker_id
        self.config_path = f"/tmp/tinyproxy_{worker_id}.conf"
        self.pid_path = f"/tmp/tinyproxy_{worker_id}.pid"
        self.log_path = f"/tmp/tinyproxy_{worker_id}.log"
        self.process = None

    async def start(self, whitelist: List[str]):
        """啟動 Proxy 並設定白名單"""
        # 1. 產生 Config
        config_content = f"""
Port {self.port}
Listen 127.0.0.1
Timeout 600
PidFile "{self.pid_path}"
LogFile "{self.log_path}"
LogLevel Info
MaxClients 100
MinSpareServers 1
MaxSpareServers 5
StartServers 1
Allow 127.0.0.1
"""
        # 加入白名單過濾
        if whitelist:
            filter_path = f"/tmp/tinyproxy_{self.worker_id}.filter"
            async with aiofiles.open(filter_path, "w") as f:
                for domain in whitelist:
                    await f.write(f"{domain}\n")
            
            config_content += f"""
Filter "{filter_path}"
FilterURLs On
FilterExtended On
FilterDefaultDeny Yes
"""
        
        async with aiofiles.open(self.config_path, "w") as f:
            await f.write(config_content)

        # 2. 啟動 TinyProxy
        try:
            self.process = await asyncio.create_subprocess_exec(
                "tinyproxy", "-d", "-c", self.config_path,
                stdout=asyncio.subprocess.DEVNULL,
                stderr=asyncio.subprocess.DEVNULL
            )
            # 等待啟動
            await asyncio.sleep(0.5)
            logger.info(f"Worker-{self.worker_id}: Proxy started on port {self.port}")
        except Exception as e:
            logger.error(f"Worker-{self.worker_id}: Failed to start proxy: {e}")

    async def get_logs(self) -> str:
        """讀取 Proxy Log"""
        if os.path.exists(self.log_path):
            try:
                async with aiofiles.open(self.log_path, "r", errors="ignore") as f:
                    return await f.read()
            except Exception:
                return ""
        return ""

    async def stop(self):
        """停止 Proxy"""
        if self.process:
            try:
                self.process.terminate()
                await self.process.wait()
            except Exception:
                pass
            self.process = None
        
        # Cleanup files
        for p in [self.config_path, self.pid_path, f"/tmp/tinyproxy_{self.worker_id}.filter", self.log_path]:
            if os.path.exists(p):
                try:
                    os.remove(p)
                except: pass

class SidecarManager:
    """管理 Sidecar 容器"""
    
    # 白名單：只允許這些 Image
    ALLOWED_IMAGES = {
        # 快取 / Key-Value
        "redis:alpine",
        "redis:7-alpine",
        "memcached:alpine",
        
        # 關聯式資料庫
        "mysql:8.0",
        "mysql:5.7",
        "mariadb:10",
        "postgres:14-alpine",
        "postgres:15-alpine",
        
        # NoSQL 資料庫
        "mongo:6.0",
        "mongo:7.0",
        
        # 訊息佇列
        "rabbitmq:3-alpine",
        "nats:alpine",
        
        # 物件儲存
        "minio/minio",
        
        # 時序/分析資料庫
        "influxdb:2",
        "clickhouse/clickhouse-server",
    }
    
    # 常用映像檔（建議預先拉取）
    PRELOAD_IMAGES = {
        "redis:alpine",
        "mysql:8.0",
        "postgres:14-alpine",
        "mongo:6.0",
    }

    def __init__(self, worker_id: int):
        self.worker_id = worker_id
        self.client = None
        self.container = None
        self.container_name = f"sandbox_sidecar_{worker_id}"
        try:
            self.client = docker.from_env()
        except Exception:
            logger.warning("Docker client not available, Sidecar features will fail.")
    
    @classmethod
    async def preload_common_images(cls):
        """預先拉取常用映像檔（系統啟動時呼叫）"""
        try:
            client = docker.from_env()
            loop = asyncio.get_running_loop()
            
            for image in cls.PRELOAD_IMAGES:
                try:
                    client.images.get(image)
                    logger.info(f"[Preload] Image {image} already exists")
                except docker.errors.ImageNotFound:
                    logger.info(f"[Preload] Pulling {image}...")
                    await loop.run_in_executor(None, lambda img=image: client.images.pull(img))
                    logger.info(f"[Preload] {image} pulled successfully")
        except Exception as e:
            logger.warning(f"[Preload] Failed to preload images: {e}")

    async def start(self, image: str):
        """啟動 Sidecar 容器"""
        if not self.client:
            raise RuntimeError("Docker client not initialized")

        # 1. 白名單檢查
        if image not in self.ALLOWED_IMAGES:
            raise ValueError(f"Image '{image}' is not allowed. Allowed: {self.ALLOWED_IMAGES}")

        try:
            # Get current container ID (hostname in Docker)
            hostname = socket.gethostname()
            
            loop = asyncio.get_running_loop()
            
            # 檢查映像檔是否已存在，只有不存在時才拉取
            try:
                self.client.images.get(image)
                logger.info(f"Sidecar image {image} already exists locally")
            except docker.errors.ImageNotFound:
                logger.info(f"Pulling sidecar image: {image} (first time, may take a while)")
                await loop.run_in_executor(None, lambda: self.client.images.pull(image))
                logger.info(f"Sidecar image {image} pulled successfully")
            
            # Remove existing if any
            try:
                old = self.client.containers.get(self.container_name)
                old.remove(force=True)
            except docker.errors.NotFound:
                pass
            
            # Run container attached to current network namespace
            logger.info(f"Starting sidecar {image} attached to {hostname}")
            
            # 設定環境變數 (從環境變數讀取資料庫密碼)
            sidecar_db_password = os.getenv("SIDECAR_DB_PASSWORD", "sandbox_db_pass")
            env_vars = {}
            if "mysql" in image or "mariadb" in image:
                env_vars["MYSQL_ROOT_PASSWORD"] = sidecar_db_password
                env_vars["MYSQL_ALLOW_EMPTY_PASSWORD"] = "yes"
            elif "postgres" in image:
                env_vars["POSTGRES_PASSWORD"] = sidecar_db_password
                env_vars["POSTGRES_HOST_AUTH_METHOD"] = "trust"
            elif "mongo" in image:
                pass  # MongoDB 預設不需密碼
            elif "minio" in image:
                env_vars["MINIO_ROOT_USER"] = "minioadmin"
                env_vars["MINIO_ROOT_PASSWORD"] = sidecar_db_password
            elif "rabbitmq" in image:
                env_vars["RABBITMQ_DEFAULT_USER"] = "guest"
                env_vars["RABBITMQ_DEFAULT_PASS"] = "guest"
            elif "clickhouse" in image:
                env_vars["CLICKHOUSE_USER"] = "default"
                env_vars["CLICKHOUSE_PASSWORD"] = ""

            self.container = await loop.run_in_executor(None, lambda: self.client.containers.run(
                image,
                detach=True,
                name=self.container_name,
                network_mode=f"container:{hostname}", # Share network stack
                auto_remove=True,
                environment=env_vars,
                # 資源限制
                mem_limit="512m",
                nano_cpus=500000000 # 0.5 CPU
            ))
            
            # 根據服務類型調整等待時間
            startup_time = self._get_startup_time(image)
            logger.info(f"Waiting {startup_time}s for {image} to start...")
            await asyncio.sleep(startup_time)
            
            # Check if container is still running
            self.container.reload()
            if self.container.status != 'running':
                logs = self.container.logs().decode('utf-8', errors='ignore')
                raise RuntimeError(f"Sidecar container exited prematurely. Logs:\n{logs}")
            
        except Exception as e:
            logger.error(f"Failed to start sidecar: {e}")
            raise

    def _get_startup_time(self, image: str) -> int:
        """根據服務類型返回適當的啟動等待時間（秒）"""
        # 輕量服務：快速啟動
        if any(x in image for x in ["redis", "memcached", "nats"]):
            return 2
        # 中型服務
        if any(x in image for x in ["mongo", "postgres", "mariadb", "rabbitmq", "influxdb"]):
            return 5
        # 重型服務：需要較長初始化
        if any(x in image for x in ["mysql", "clickhouse", "minio"]):
            return 8
        # 預設
        return 5

    async def stop(self):
        """停止 Sidecar"""
        if self.container:
            try:
                loop = asyncio.get_running_loop()
                await loop.run_in_executor(None, self.container.stop)
            except Exception as e:
                logger.error(f"Error stopping sidecar: {e}")
            self.container = None

class Worker:
    def __init__(self, worker_id: int, queue, manager):
        self.worker_id = worker_id
        self.queue = queue
        self.manager = manager
        self.box_id = worker_id # Bind Worker ID to Box ID (0 to N-1)
        self.running = False
        self.current_job: Job = None
        self.runner = IsolateRunner() 
        self.is_busy = False # 狀態追蹤
        self.proxy = ProxyManager(worker_id)
        self.sidecar = SidecarManager(worker_id)
        
    async def run(self):
        """Worker 主迴圈"""
        self.running = True
        logger.info(f"Worker-{self.worker_id} started (Box ID: {self.box_id})")
        
        # 初始化 Box
        try:
            box_path = await self.runner._init_box(self.box_id, use_cg=False) 
            logger.info(f"Worker-{self.worker_id} box initialized at {box_path}")
        except Exception as e:
            logger.error(f"Worker-{self.worker_id} failed to init box: {e}")
            return

        while self.running:
            try:
                # 1. 從隊列獲取任務
                try:
                    job = await asyncio.wait_for(self.queue.pop(), timeout=1.0)
                except asyncio.TimeoutError:
                    continue
                
                self.is_busy = True
                self.current_job = job
                logger.info(f"Worker-{self.worker_id} processing job {job.submission_id}")
                
                result_data = {"status": "XX", "message": "Unknown Error"}
                
                # Network Setup
                share_net = False
                env_vars = {}
                
                # Sidecar Setup
                if job.sidecar_image:
                    try:
                        await self.sidecar.start(job.sidecar_image)
                        share_net = True # Must share net to access sidecar on localhost
                        logger.info(f"Job {job.submission_id}: Sidecar {job.sidecar_image} started")
                    except Exception as e:
                        logger.error(f"Job {job.submission_id}: Sidecar failed: {e}")
                        result_data = {"status": "XX", "message": f"Sidecar failed: {e}"}
                        # Skip execution if sidecar fails? Yes.
                        # But we are inside the loop, so we need to handle flow.
                        # Let's set a flag or raise exception to jump to finally
                        raise RuntimeError(f"Sidecar failed: {e}")

                if job.allow_network:
                    share_net = True
                    if job.network_whitelist:
                        await self.proxy.start(job.network_whitelist)
                        proxy_url = f"http://127.0.0.1:{self.proxy.port}"
                        env_vars = {
                            "http_proxy": proxy_url,
                            "https_proxy": proxy_url,
                            "HTTP_PROXY": proxy_url,
                            "HTTPS_PROXY": proxy_url
                        }
                        logger.info(f"Job {job.submission_id}: Network enabled with whitelist proxy")
                    else:
                        logger.warning(f"Job {job.submission_id}: Full network access enabled (No whitelist)")

                try:
                    # 2. 準備檔案 & 編譯
                    compile_success = True
                    executable_cmd = []
                    run_files_base = []
                    compile_info = {"status": "OK", "message": "No compilation needed for this language"}
                    source_code = "" 
                    
                    lang = job.language.lower()
                    
                    # 判斷是否有題包
                    has_problem_package_for_compile = job.problem_id is not None
                    if has_problem_package_for_compile:
                        problem_dir = os.path.join("test_data", "problems", job.problem_id)
                    else:
                        problem_dir = None

                    # [Step 2.1] Load Teacher Provided Files (Scenario 3)
                    provided_files_data = [] # List of (filename, content_bytes)
                    has_provided_makefile = False
                    has_provided_main = False
                    
                    if problem_dir:
                        meta_path = os.path.join(problem_dir, "meta.json")
                    else:
                        meta_path = None
                    
                    if meta_path and os.path.exists(meta_path):
                        try:
                            async with aiofiles.open(meta_path, "r") as f:
                                meta_json = json.loads(await f.read())
                                p_files = meta_json.get("provided_files", [])
                                for pf in p_files:
                                    pf_path = os.path.join(problem_dir, pf)
                                    if os.path.exists(pf_path):
                                        if pf == "Makefile" or pf == "makefile": has_provided_makefile = True
                                        if pf == "main.c" or pf == "main.cpp": has_provided_main = True
                                        
                                        async with aiofiles.open(pf_path, "rb") as f_p:
                                            provided_files_data.append((pf, await f_p.read()))
                        except Exception as e:
                            logger.warning(f"Failed to load provided files for {job.problem_id}: {e}")

                    # 處理不同模式
                    if job.mode == "package":
                        # Package 模式：題包中包含 submission 代碼、testcase、checker
                        # file_path 指向 submission 文件
                        # problem_id 格式為 "package_{submission_id}"
                        
                        # 讀取 submission 代碼
                        if os.path.exists(job.file_path):
                            async with aiofiles.open(job.file_path, "r", encoding="utf-8", errors="ignore") as f:
                                source_code = await f.read()
                        else:
                            raise FileNotFoundError(f"Submission file not found: {job.file_path}")
                        
                        # 根據語言處理編譯
                        if lang in ["c", "cpp"]:
                            student_filename = "main.cpp" if lang == "cpp" else "main.c"
                            files = [(student_filename, source_code)]
                            compiler = "/usr/bin/g++" if lang == "cpp" else "/usr/bin/gcc"
                            
                            compile_result = await self.runner.execute(
                                box_id=self.box_id,
                                box_path=box_path,
                                command=[compiler, student_filename, "-o", "main"],
                                files=files,
                                limits={"time_limit_sec": 10, "wall_time_limit_sec": 20, "process_limit": 20},
                                capture_output_to_files=False,
                                share_net=share_net,
                                env=env_vars
                            )
                            
                            compile_info = {
                                "status": "OK" if compile_result['status'] == 'OK' else 'CE',
                                "stdout": compile_result.get('stdout', ''),
                                "stderr": compile_result.get('stderr', ''),
                                "time_used": compile_result.get('time_used_sec', 0),
                                "memory_used": compile_result.get('memory_used_kb', 0)
                            }
                            
                            if compile_result['status'] != 'OK':
                                result_data = compile_result
                                result_data['status'] = 'CE'
                                result_data['compile_info'] = compile_info
                                compile_success = False
                            else:
                                executable_cmd = ["./main"]
                                run_files_base = []
                                
                        elif lang == "python":
                            run_files_base = [("main.py", source_code)]
                            executable_cmd = ["/usr/bin/python3", "main.py"]
                            compile_info = {"status": "OK", "message": "Python interpreted"}
                        else:
                            result_data = {"status": "XX", "message": f"Language {lang} not supported in package mode"}
                            compile_success = False
                        
                        # 執行測試用例
                        if compile_success:
                            # Package 模式下，testcase 在上傳的 ZIP 的 extracted 目錄中
                            # 通過 job.file_path 我們可以找到 work_dir
                            work_dir = os.path.dirname(os.path.dirname(job.file_path))  # /tmp/sandbox/packages/{submission_id}
                            testcase_dir = os.path.join(work_dir, "extracted", "testcase")
                            
                            if not os.path.exists(testcase_dir):
                                result_data = {"status": "XX", "message": "Testcase directory not found"}
                                compile_success = False
                            else:
                                # 獲取所有測試用例
                                import glob
                                in_files = sorted(glob.glob(os.path.join(testcase_dir, "*.in")))
                                
                                if not in_files:
                                    result_data = {"status": "XX", "message": "No test cases found"}
                                    compile_success = False
                                else:
                                    # 執行每個測試用例
                                    total_score = 0
                                    cases_results = []
                                    final_status = "AC"
                                    
                                    for in_file in in_files:
                                        case_name = os.path.basename(in_file).replace(".in", "")
                                        out_file = in_file.replace(".in", ".out")
                                        
                                        # 讀取輸入
                                        async with aiofiles.open(in_file, "r") as f:
                                            stdin_content = await f.read()
                                        
                                        # 讀取期望輸出
                                        expected_output = ""
                                        if os.path.exists(out_file):
                                            async with aiofiles.open(out_file, "r") as f:
                                                expected_output = (await f.read()).strip()
                                        
                                        # 執行
                                        limits = {
                                            "time_limit_sec": job.request_data.limits.time_limit_sec,
                                            "wall_time_limit_sec": job.request_data.limits.wall_time_limit_sec,
                                            "memory_limit_kb": job.request_data.limits.memory_limit_kb,
                                            "process_limit": 60
                                        }
                                        
                                        run_result = await self.runner.execute(
                                            box_id=self.box_id,
                                            box_path=box_path,
                                            command=executable_cmd,
                                            files=run_files_base,
                                            stdin=stdin_content,
                                            limits=limits,
                                            capture_output_to_files=True,
                                            share_net=share_net,
                                            env=env_vars
                                        )
                                        
                                        case_status = run_result.get('status', 'XX')
                                        message = run_result.get('message', '')
                                        
                                        # Checker 驗證
                                        if case_status == 'OK':
                                            actual_output = run_result.get('stdout', '').strip()
                                            
                                            if job.use_checker and job.checker_file_path:
                                                # 使用 Checker（在 Isolate 沙盒中執行）
                                                try:
                                                    # 準備 checker 文件
                                                    async with aiofiles.open(job.checker_file_path, 'rb') as f:
                                                        checker_binary = await f.read()
                                                    
                                                    # 準備輸入文件
                                                    # 使用原始測試用例文件名（如 0001.in, 0001.out）
                                                    in_filename = f"{case_name}.in"
                                                    out_filename = f"{case_name}.out"
                                                    
                                                    checker_files = [
                                                        ('checker', checker_binary),
                                                        (in_filename, stdin_content.encode('utf-8')),
                                                        (out_filename, actual_output.encode('utf-8')),
                                                        ('answer', expected_output.encode('utf-8'))
                                                    ]
                                                    
                                                    # 在 Isolate 中執行 checker
                                                    checker_result = await self.runner.execute(
                                                        box_id=self.box_id,
                                                        box_path=box_path,
                                                        command=['./checker', in_filename, out_filename, 'answer'],
                                                        files=checker_files,
                                                        stdin='',
                                                        limits={
                                                            'time_limit_sec': 5.0,
                                                            'wall_time_limit_sec': 10.0,
                                                            'memory_limit_kb': 262144,
                                                            'process_limit': 10
                                                        },
                                                        capture_output_to_files=True,
                                                        share_net=False,
                                                        env={}
                                                    )
                                                    
                                                    # 判斷結果
                                                    if checker_result['status'] == 'OK' and checker_result.get('exitcode', 1) == 0:
                                                        case_status = 'AC'
                                                    elif checker_result['status'] == 'OK':
                                                        case_status = 'WA'
                                                        checker_msg = checker_result.get('stderr', '').strip() or checker_result.get('stdout', '').strip()
                                                        message = f"Checker: {checker_msg[:100]}"
                                                    else:
                                                        case_status = 'XX'
                                                        message = f"Checker {checker_result['status']}: {checker_result.get('message', '')}"
                                                        
                                                except Exception as e:
                                                    case_status = 'XX'
                                                    message = f"Checker error: {str(e)}"
                                            else:
                                                # 默認 diff 比對
                                                if actual_output == expected_output:
                                                    case_status = 'AC'
                                                else:
                                                    case_status = 'WA'
                                                    message = f"Output mismatch"
                                        
                                        cases_results.append({
                                            "case_name": case_name,
                                            "status": case_status,
                                            "time": run_result.get('time_used_sec'),
                                            "memory": run_result.get('memory_used_kb'),
                                            "message": message
                                        })
                                        
                                        if case_status == 'AC':
                                            total_score += 1
                                        else:
                                            if final_status == 'AC':
                                                final_status = case_status
                                    
                                    # 計算總分（百分比）
                                    score_percentage = int((total_score / len(in_files)) * 100)
                                    
                                    result_data = {
                                        "status": final_status,
                                        "score": score_percentage,
                                        "tasks": [{
                                            "task_id": 0,
                                            "score": score_percentage,
                                            "cases": cases_results
                                        }],
                                        "compile_info": compile_info,
                                        "time_used_sec": max([c['time'] for c in cases_results] or [0]),
                                        "memory_used_kb": max([c['memory'] for c in cases_results] or [0]),
                                        "message": f"Passed {total_score}/{len(in_files)} test cases"
                                    }
                    
                    elif job.mode == "zip":
                        # 讀取 ZIP 內容
                        if os.path.exists(job.file_path):
                            async with aiofiles.open(job.file_path, "rb") as f:
                                zip_content = await f.read()
                        else:
                            raise FileNotFoundError(f"Source file not found: {job.file_path}")

                        # 檢查 ZIP 內容以決定編譯策略
                        has_makefile = False
                        has_main_cpp = False
                        has_main_c = False
                        
                        try:
                            with zipfile.ZipFile(io.BytesIO(zip_content)) as z:
                                namelist = z.namelist()
                                if "Makefile" in namelist or "makefile" in namelist:
                                    has_makefile = True
                                if "main.cpp" in namelist:
                                    has_main_cpp = True
                                if "main.c" in namelist:
                                    has_main_c = True
                        except zipfile.BadZipFile:
                            raise ValueError("Invalid ZIP file")

                        # 設定檔案列表 (IsolateRunner 會自動解壓 .zip)
                        # 合併題目提供的檔案 (Teacher's files) 與學生的 ZIP
                        files = provided_files_data + [("source.zip", zip_content)]
                        
                        # 優先使用 ZIP 內的 Makefile，其次是題目提供的 Makefile
                        use_make = has_makefile or has_provided_makefile
                        
                        if use_make:
                            logger.info(f"Job {job.submission_id}: Makefile detected (Student: {has_makefile}, Teacher: {has_provided_makefile})")
                            # 使用 make 編譯
                            compile_result = await self.runner.execute(
                                box_id=self.box_id,
                                box_path=box_path,
                                command=["/usr/bin/make"],
                                files=files,
                                limits={"time_limit_sec": 30, "wall_time_limit_sec": 60, "process_limit": 50},
                                capture_output_to_files=False,
                                share_net=share_net,
                                env=env_vars
                            )
                            
                            compile_info = {
                                "status": "OK" if compile_result['status'] == 'OK' else 'CE',
                                "stdout": compile_result.get('stdout', ''),
                                "stderr": compile_result.get('stderr', ''),
                                "time_used": compile_result.get('time_used_sec', 0),
                                "memory_used": compile_result.get('memory_used_kb', 0)
                            }

                            if compile_result['status'] != 'OK':
                                result_data = compile_result
                                result_data['status'] = 'CE'
                                result_data['compile_info'] = compile_info
                                compile_success = False
                            else:
                                # 假設 Makefile 產生的執行檔名為 main
                                executable_cmd = ["./main"]
                                run_files_base = [] 

                        elif has_main_cpp or has_main_c or has_provided_main:
                            # ZIP 但沒有 Makefile，嘗試直接編譯 main.cpp/c (包含題目提供的)
                            # 這裡簡單假設如果是 C++ 就用 g++，否則 gcc
                            # 如果題目提供了 main.cpp，我們就用 g++
                            is_cpp = has_main_cpp or (has_provided_main and "cpp" in [f[0] for f in provided_files_data])
                            
                            compiler = "/usr/bin/g++" if is_cpp else "/usr/bin/gcc"
                            # 編譯所有 .c / .cpp 檔案
                            src_pattern = "*.cpp" if is_cpp else "*.c"
                            
                            # 注意：Isolate 的 execute command 是 execv 風格，不支援 shell wildcard expansion (*.c)
                            # 所以我們需要用 shell 來執行編譯指令
                            compile_cmd = ["/bin/sh", "-c", f"{compiler} {src_pattern} -o main"]
                            
                            compile_result = await self.runner.execute(
                                box_id=self.box_id,
                                box_path=box_path,
                                command=compile_cmd,
                                files=files, # 傳入 ZIP + Provided Files
                                limits={"time_limit_sec": 10, "wall_time_limit_sec": 20, "process_limit": 20},
                                capture_output_to_files=False,
                                share_net=share_net,
                                env=env_vars
                            )
                            
                            compile_info = {
                                "status": "OK" if compile_result['status'] == 'OK' else 'CE',
                                "stdout": compile_result.get('stdout', ''),
                                "stderr": compile_result.get('stderr', ''),
                            }
                            
                            if compile_result['status'] != 'OK':
                                result_data = compile_result
                                result_data['status'] = 'CE'
                                result_data['compile_info'] = compile_info
                                compile_success = False
                            else:
                                executable_cmd = ["./main"]
                                run_files_base = []
                        else:
                            result_data = {"status": "CE", "message": "No Makefile or main.cpp/c found in ZIP or Provided Files"}
                            compile_success = False

                    elif lang in ["c", "cpp", "python"]:
                        # Normal Mode (Single File)
                        if os.path.exists(job.file_path):
                            async with aiofiles.open(job.file_path, "r", encoding="utf-8", errors="ignore") as f:
                                source_code = await f.read()
                        else:
                            raise FileNotFoundError(f"Source file not found: {job.file_path}")

                        if lang in ["c", "cpp"]:
                            # 決定學生檔案名稱
                            # 如果題目提供了 main.c/cpp，則學生檔案命名為 solution.c/cpp
                            # 否則學生檔案命名為 main.c/cpp
                            student_filename = "main.cpp" if lang == "cpp" else "main.c"
                            if has_provided_main:
                                student_filename = "solution.cpp" if lang == "cpp" else "solution.c"
                            
                            files = provided_files_data + [(student_filename, source_code)] if source_code else provided_files_data
                            
                            # 決定編譯方式
                            if has_provided_makefile:
                                # 使用題目提供的 Makefile
                                compile_cmd = ["/usr/bin/make"]
                            else:
                                # 編譯所有相關檔案
                                compiler = "/usr/bin/g++" if lang == "cpp" else "/usr/bin/gcc"
                                src_pattern = "*.cpp" if lang == "cpp" else "*.c"
                                # 混合 C 和 C++ 的情況比較複雜，這裡暫時假設語言一致
                                compile_cmd = ["/bin/sh", "-c", f"{compiler} {src_pattern} -o main"]

                            # Compile
                            compile_result = await self.runner.execute(
                                box_id=self.box_id,
                                box_path=box_path,
                                command=compile_cmd,
                                files=files,
                                limits={"time_limit_sec": 10, "wall_time_limit_sec": 20, "process_limit": 20},
                                capture_output_to_files=False,
                                share_net=share_net,
                                env=env_vars
                            )
                            
                            compile_info = {
                                "status": "OK" if compile_result['status'] == 'OK' else 'CE',
                                "stdout": compile_result.get('stdout', ''),
                                "stderr": compile_result.get('stderr', ''),
                                "time_used": compile_result.get('time_used_sec', 0),
                                "memory_used": compile_result.get('memory_used_kb', 0)
                            }

                            if compile_result['status'] != 'OK':
                                logger.info(f"Job {job.submission_id} Compilation Failed")
                                result_data = compile_result
                                result_data['status'] = 'CE'
                                result_data['compile_info'] = compile_info
                                compile_success = False
                            else:
                                executable_cmd = ["./main"]
                                
                        elif lang == "python":
                            filename = "main.py"
                            # 如果是 Python，通常是單檔，但如果題目有提供 lib.py，也需要一起放進去
                            # Python 不需要編譯，直接執行
                            # 如果題目提供了 main.py (Entry point)，則學生檔案可能叫 solution.py
                            # 但 Python 執行需要指定 Entry point
                            
                            entry_point = "main.py"
                            student_filename = "main.py"
                            
                            # 檢查是否有提供的 main.py
                            provided_main_py = any(f[0] == "main.py" for f in provided_files_data)
                            
                            if provided_main_py:
                                student_filename = "solution.py"
                                entry_point = "main.py"
                            
                            run_files_base = provided_files_data + [(student_filename, source_code)]
                            executable_cmd = ["/usr/bin/python3", entry_point]
                            compile_info = {"status": "OK", "message": "Python interpreted"}
                    else:
                        result_data = {"status": "XX", "message": f"Language {job.language} not fully supported"}
                        compile_info = {"status": "XX", "message": f"Language {job.language} not supported"}
                        compile_success = False

                    # Execution Phase
                    if compile_success:
                        # 判斷是否為純自定義測試模式（沒有 problem_id）
                        has_problem_package = job.problem_id is not None
                        
                        if has_problem_package:
                            problem_dir = os.path.join("test_data", "problems", job.problem_id)
                            meta_path = os.path.join(problem_dir, "meta.json")
                        else:
                            problem_dir = None
                            meta_path = None
                        
                        # Check if we should run in "Single Case Mode"
                        # Selftest (is_selftest=True) 永遠走單一執行模式，直接回傳 stdout/stderr
                        # 其他情況：有 stdin 或沒有題包時也走單一執行
                        is_selftest = getattr(job, 'is_selftest', False)
                        if is_selftest or job.stdin is not None or not has_problem_package:
                            # Single Case Execution
                            limits = {
                                "time_limit_sec": job.request_data.limits.time_limit_sec,
                                "wall_time_limit_sec": job.request_data.limits.wall_time_limit_sec,
                                "memory_limit_kb": job.request_data.limits.memory_limit_kb,
                                "process_limit": 60
                            }
                            
                            run_result = await self.runner.execute(
                                box_id=self.box_id,
                                box_path=box_path,
                                command=executable_cmd,
                                files=run_files_base,
                                stdin=job.stdin,
                                limits=limits,
                                capture_output_to_files=True,
                                share_net=share_net,
                                env=env_vars
                            )
                            
                            case_status = run_result.get('status', 'XX')
                            message = run_result.get('message', '')
                            actual_output = run_result.get('stdout', '').strip()
                            
                            # Checker Logic for Single Case
                            if case_status == 'OK':
                                if job.expected_output:
                                    expected = job.expected_output.strip()
                                    if actual_output == expected:
                                        case_status = 'AC'
                                    else:
                                        case_status = 'WA'
                                        message = f"Expected: {expected[:50]}..., Got: {actual_output[:50]}..."
                                else:
                                    # If no expected output, we assume OK is enough or check return code
                                    pass

                            result_data = {
                                "status": case_status,
                                "score": 100 if case_status == 'AC' else 0,
                                "tasks": [],
                                "time_used_sec": run_result.get('time_used_sec'),
                                "memory_used_kb": run_result.get('memory_used_kb'),
                                "message": message,
                                "stdout": actual_output,
                                "stderr": run_result.get('stderr', ''),
                                "compile_info": compile_info
                            }

                        elif meta_path and os.path.exists(meta_path):
                            # Multi-task Evaluation
                            async with aiofiles.open(meta_path, "r") as f:
                                meta = json.loads(await f.read())
                            
                            total_score = 0
                            tasks_results = []
                            final_status = "AC"
                            
                            # 檢查是否使用新格式（testcases 陣列）或舊格式（tasks 陣列）
                            testcases = meta.get("testcases", [])
                            tasks = meta.get("tasks", [])
                            
                            if testcases:
                                # 新格式：後端提供的 testcases 列表
                                # 格式: [{"no": 1, "stem": "0001", "in": "0001.in", "out": "0001.out"}, ...]
                                task_count = len(testcases)
                                score_per_case = 100 // task_count if task_count > 0 else 100
                                
                                task_passed = True
                                cases_results = []
                                
                                for tc in testcases:
                                    case_name = tc.get("stem", "")
                                    in_file = tc.get("in", f"{case_name}.in")
                                    out_file = tc.get("out", f"{case_name}.out")
                                    
                                    # 測資直接在題目根目錄（無 testcase 子目錄）
                                    input_file = os.path.join(problem_dir, in_file)
                                    output_file = os.path.join(problem_dir, out_file)
                                    
                                    if not os.path.exists(input_file):
                                        cases_results.append({
                                            "case_name": case_name,
                                            "status": "SKIPPED",
                                            "time": 0,
                                            "memory": 0,
                                            "message": f"Input file not found: {in_file}"
                                        })
                                        task_passed = False
                                        if final_status == "AC":
                                            final_status = "XX"
                                        continue
                                    
                                    # Read Input
                                    async with aiofiles.open(input_file, "r") as f:
                                        stdin_content = await f.read()
                                    
                                    # Read Expected Output
                                    expected_output = ""
                                    if os.path.exists(output_file):
                                        async with aiofiles.open(output_file, "r") as f:
                                            expected_output = (await f.read()).strip()
                                    
                                    # 使用 job 的 limits 或預設值
                                    isolate_limits = {
                                        "time_limit_sec": job.request_data.limits.time_limit_sec,
                                        "wall_time_limit_sec": job.request_data.limits.wall_time_limit_sec,
                                        "memory_limit_kb": job.request_data.limits.memory_limit_kb,
                                        "process_limit": 60
                                    }
                                    
                                    # Execute
                                    run_result = await self.runner.execute(
                                        box_id=self.box_id,
                                        box_path=box_path,
                                        command=executable_cmd,
                                        files=run_files_base,
                                        stdin=stdin_content,
                                        limits=isolate_limits,
                                        capture_output_to_files=True,
                                        share_net=share_net,
                                        env=env_vars
                                    )
                                    
                                    # Verify - 確保 case_status 總是被初始化
                                    case_status = run_result.get('status', 'XX')
                                    message = run_result.get('message', '')
                                    llm_feedback = None  # LLM judge 回饋
                                    case_score = 0  # 單測資得分
                                    
                                    if case_status == 'OK':
                                        actual_output = run_result.get('stdout', '').strip()
                                        
                                        # 判題邏輯：檢查 judge_type
                                        judge_type = meta.get("judge_type", "diff")  # 預設為 diff
                                        
                                        if judge_type == "llm":
                                            # LLM-as-Judge 判題
                                            try:
                                                llm_result = await llm_judge_case(
                                                    meta=meta,
                                                    student_output=actual_output,
                                                    stdin_content=stdin_content,
                                                    expected_output=expected_output
                                                )
                                                case_status = llm_result["verdict"]
                                                case_score = llm_result["score"]
                                                message = llm_result["reason"]
                                                llm_feedback = llm_result.get("llm_feedback")
                                                logger.info(f"LLM judge result for {case_name}: {case_status}, score={case_score}")
                                            except Exception as e:
                                                logger.error(f"LLM judge error for {case_name}: {e}")
                                                case_status = 'JE'
                                                message = f"LLM Judge Error: {str(e)[:100]}"
                                        
                                        # Checker Logic (原有邏輯)
                                        elif job.use_checker:
                                            if job.checker_file_path == "diff":
                                                if actual_output == expected_output:
                                                    case_status = 'AC'
                                                else:
                                                    case_status = 'WA'
                                                    message = f"Expected: {expected_output[:50]}..., Got: {actual_output[:50]}..."
                                            elif job.checker_file_path and os.path.exists(job.checker_file_path):
                                                # Custom Checker
                                                try:
                                                    async with aiofiles.open(job.checker_file_path, 'rb') as f:
                                                        checker_binary = await f.read()
                                                    
                                                    in_filename = f"{case_name}.in"
                                                    out_filename = f"{case_name}.out"
                                                    
                                                    if job.checker_file_path.endswith('.py'):
                                                        checker_files = [
                                                            ('checker.py', checker_binary),
                                                            (in_filename, stdin_content.encode('utf-8')),
                                                            (out_filename, actual_output.encode('utf-8')),
                                                            ('answer', expected_output.encode('utf-8'))
                                                        ]
                                                        checker_cmd = ['/usr/bin/python3', 'checker.py', in_filename, out_filename, 'answer']
                                                    else:
                                                        checker_files = [
                                                            ('checker', checker_binary),
                                                            (in_filename, stdin_content.encode('utf-8')),
                                                            (out_filename, actual_output.encode('utf-8')),
                                                            ('answer', expected_output.encode('utf-8'))
                                                        ]
                                                        checker_cmd = ['./checker', in_filename, out_filename, 'answer']
                                                    
                                                    checker_result = await self.runner.execute(
                                                        box_id=self.box_id,
                                                        box_path=box_path,
                                                        command=checker_cmd,
                                                        files=checker_files,
                                                        stdin='',
                                                        limits={
                                                            'time_limit_sec': 5.0,
                                                            'wall_time_limit_sec': 10.0,
                                                            'memory_limit_kb': 262144,
                                                            'process_limit': 10
                                                        },
                                                        capture_output_to_files=True,
                                                        share_net=False,
                                                        env={}
                                                    )
                                                    
                                                    if checker_result['status'] == 'OK' and checker_result.get('exitcode', 1) == 0:
                                                        case_status = 'AC'
                                                    elif checker_result['status'] == 'OK':
                                                        case_status = 'WA'
                                                        checker_msg = checker_result.get('stderr', '').strip() or checker_result.get('stdout', '').strip()
                                                        message = f"Checker: {checker_msg[:100]}"
                                                    else:
                                                        case_status = 'XX'
                                                        message = f"Checker {checker_result['status']}: {checker_result.get('message', '')}"
                                                except Exception as e:
                                                    case_status = 'XX'
                                                    message = f"Checker Error: {str(e)}"
                                            else:
                                                case_status = 'XX'
                                                message = "Checker not found"
                                        else:
                                            # 預設 diff 比對
                                            if actual_output == expected_output:
                                                case_status = 'AC'
                                            else:
                                                case_status = 'WA'
                                                message = f"Expected: {expected_output[:50]}..., Got: {actual_output[:50]}..."
                                    
                                    # 構建 case 結果
                                    case_result_entry = {
                                        "case_name": case_name,
                                        "status": case_status,
                                        "time": run_result.get('time_used_sec'),
                                        "memory": run_result.get('memory_used_kb'),
                                        "message": message
                                    }
                                    
                                    # 如果有 LLM feedback，加入結果
                                    if llm_feedback:
                                        case_result_entry["llm_feedback"] = llm_feedback
                                        case_result_entry["llm_score"] = case_score
                                    
                                    cases_results.append(case_result_entry)
                                    
                                    if case_status == 'AC':
                                        total_score += score_per_case
                                    else:
                                        task_passed = False
                                        if final_status == 'AC':
                                            final_status = case_status
                                
                                tasks_results.append({
                                    "task_id": 0,
                                    "score": total_score,
                                    "cases": cases_results
                                })
                                
                                result_data = {
                                    "status": final_status,
                                    "score": total_score,
                                    "tasks": tasks_results,
                                    "compile_info": compile_info,
                                    "time_used_sec": max([c['time'] for c in cases_results if c.get('time')] or [0]),
                                    "memory_used_kb": max([c['memory'] for c in cases_results if c.get('memory')] or [0]),
                                    "message": f"Score: {total_score}"
                                }
                            
                            elif tasks:
                                # tasks 格式：從 ZIP meta.json 取得
                                # 格式: [{"caseCount": 5, "memoryLimit": 512, "taskScore": 50, "timeLimit": 2000}]
                                # 測資檔名: 0001.in, 0002.in, ... (4位數純數字序號)
                                
                                global_case_idx = 0  # 全域測資編號
                                
                                for task_idx, task in enumerate(tasks):
                                    task_score = task.get("taskScore", 0)
                                    case_count = task.get("caseCount", 0)
                                    task_passed = True
                                    cases_results = []
                                    
                                    # 取得此 task 的 limits
                                    time_limit_ms = task.get("timeLimit", 2000)  # 毫秒
                                    memory_limit_mb = task.get("memoryLimit", 256)  # MB
                                    memory_limit_kb = task.get("memoryLimitKB", memory_limit_mb * 1024)  # KB
                                    
                                    for case_idx in range(case_count):
                                        global_case_idx += 1
                                        # 測資檔名格式: 0001.in, 0002.in, ...
                                        case_name = f"{global_case_idx:04d}"
                                        input_file = os.path.join(problem_dir, f"{case_name}.in")
                                        
                                        if not os.path.exists(input_file):
                                            cases_results.append({
                                                "case_name": case_name, 
                                                "status": "SKIPPED", 
                                                "time": 0, 
                                                "memory": 0, 
                                                "message": f"Input file not found: {case_name}.in"
                                            })
                                            task_passed = False
                                            if final_status == "AC":
                                                final_status = "XX"
                                            continue
                                            
                                        output_file = os.path.join(problem_dir, f"{case_name}.out")
                                        
                                        # Read Input
                                        async with aiofiles.open(input_file, "r") as f:
                                            stdin_content = await f.read()
                                        
                                        # Read Expected Output
                                        expected_output = ""
                                        if os.path.exists(output_file):
                                            async with aiofiles.open(output_file, "r") as f:
                                                expected_output = (await f.read()).strip()
                                                
                                        # Convert task limits (使用 ZIP meta.json 的值)
                                        isolate_limits = {
                                            "time_limit_sec": time_limit_ms / 1000.0,
                                            "wall_time_limit_sec": (time_limit_ms / 1000.0) * 3.0,
                                            "memory_limit_kb": memory_limit_kb,
                                            "process_limit": 60
                                        }

                                        # Execute
                                        run_result = await self.runner.execute(
                                            box_id=self.box_id,
                                            box_path=box_path,
                                            command=executable_cmd,
                                            files=run_files_base,
                                            stdin=stdin_content,
                                            limits=isolate_limits,
                                            capture_output_to_files=True,
                                            share_net=share_net,
                                            env=env_vars
                                        )
                                        
                                        # Verify - 確保 case_status 總是被初始化
                                        case_status = run_result.get('status', 'XX')
                                        message = run_result.get('message', '')
                                        llm_feedback = None  # LLM judge 回饋
                                        case_score = 0  # 單測資得分
                                        
                                        if case_status == 'OK':
                                            actual_output = run_result.get('stdout', '').strip()
                                            
                                            # 判題邏輯：檢查 judge_type
                                            judge_type = meta.get("judge_type", "diff")  # 預設為 diff
                                            
                                            if judge_type == "llm":
                                                # LLM-as-Judge 判題
                                                try:
                                                    llm_result = await llm_judge_case(
                                                        meta=meta,
                                                        student_output=actual_output,
                                                        stdin_content=stdin_content,
                                                        expected_output=expected_output
                                                    )
                                                    case_status = llm_result["verdict"]
                                                    case_score = llm_result["score"]
                                                    message = llm_result["reason"]
                                                    llm_feedback = llm_result.get("llm_feedback")
                                                    logger.info(f"LLM judge result for {case_name}: {case_status}, score={case_score}")
                                                except Exception as e:
                                                    logger.error(f"LLM judge error for {case_name}: {e}")
                                                    case_status = 'JE'
                                                    message = f"LLM Judge Error: {str(e)[:100]}"
                                            
                                            # Checker Logic
                                            elif job.use_checker:
                                                if job.checker_file_path == "diff":
                                                    if actual_output == expected_output:
                                                        case_status = 'AC'
                                                    else:
                                                        case_status = 'WA'
                                                        message = f"Expected: {expected_output[:50]}..., Got: {actual_output[:50]}..."
                                                elif job.checker_file_path and os.path.exists(job.checker_file_path):
                                                    # Custom Checker（在 Isolate 沙盒中執行）
                                                    try:
                                                        # 準備 checker 文件
                                                        async with aiofiles.open(job.checker_file_path, 'rb') as f:
                                                            checker_binary = await f.read()
                                                        
                                                        # 使用原始測試用例文件名（如 0001.in, 0001.out）
                                                        in_filename = f"{case_name}.in"
                                                        out_filename = f"{case_name}.out"
                                                        
                                                        # 判斷 checker 類型並準備執行命令
                                                        if job.checker_file_path.endswith('.py'):
                                                            # Python checker
                                                            checker_files = [
                                                                ('checker.py', checker_binary),
                                                                (in_filename, stdin_content.encode('utf-8')),
                                                                (out_filename, actual_output.encode('utf-8')),
                                                                ('answer', expected_output.encode('utf-8'))
                                                            ]
                                                            checker_cmd = ['/usr/bin/python3', 'checker.py', in_filename, out_filename, 'answer']
                                                        else:
                                                            # Binary checker (C/C++)
                                                            checker_files = [
                                                                ('checker', checker_binary),
                                                                (in_filename, stdin_content.encode('utf-8')),
                                                                (out_filename, actual_output.encode('utf-8')),
                                                                ('answer', expected_output.encode('utf-8'))
                                                            ]
                                                            checker_cmd = ['./checker', in_filename, out_filename, 'answer']
                                                        
                                                        # 在 Isolate 中執行 checker
                                                        checker_result = await self.runner.execute(
                                                            box_id=self.box_id,
                                                            box_path=box_path,
                                                            command=checker_cmd,
                                                            files=checker_files,
                                                            stdin='',
                                                            limits={
                                                                'time_limit_sec': 5.0,
                                                                'wall_time_limit_sec': 10.0,
                                                                'memory_limit_kb': 262144,
                                                                'process_limit': 10
                                                            },
                                                            capture_output_to_files=True,
                                                            share_net=False,
                                                            env={}
                                                        )
                                                        
                                                        # 判斷結果
                                                        if checker_result['status'] == 'OK' and checker_result.get('exitcode', 1) == 0:
                                                            case_status = 'AC'
                                                        elif checker_result['status'] == 'OK':
                                                            case_status = 'WA'
                                                            checker_msg = checker_result.get('stderr', '').strip() or checker_result.get('stdout', '').strip()
                                                            message = f"Checker: {checker_msg[:100]}"
                                                        else:
                                                            case_status = 'XX'
                                                            message = f"Checker {checker_result['status']}: {checker_result.get('message', '')}"
                                                            
                                                    except Exception as e:
                                                        case_status = 'XX'
                                                        message = f"Checker Error: {str(e)}"
                                                else:
                                                    # use_checker=True 但找不到 checker 檔案
                                                    case_status = 'XX'
                                                    message = "Checker not found"
                                            else:
                                                # use_checker=False，直接字串比對
                                                if actual_output == expected_output:
                                                    case_status = 'AC'
                                                else:
                                                    case_status = 'WA'
                                                    message = f"Expected: {expected_output[:50]}..., Got: {actual_output[:50]}..."
                                    
                                        # 構建 case 結果（移出 if case_status == 'OK' 區塊，確保所有結果都被記錄）
                                        case_result_entry = {
                                            "case_name": case_name,
                                            "status": case_status,
                                            "time": run_result.get('time_used_sec'),
                                            "memory": run_result.get('memory_used_kb'),
                                            "message": message
                                        }
                                        
                                        # 如果有 LLM feedback，加入結果
                                        if llm_feedback:
                                            case_result_entry["llm_feedback"] = llm_feedback
                                            case_result_entry["llm_score"] = case_score
                                        
                                        cases_results.append(case_result_entry)
                                        
                                        if case_status != 'AC':
                                            task_passed = False
                                            if final_status == 'AC': final_status = case_status
                                        
                                if task_passed:
                                    total_score += task_score
                                    
                                tasks_results.append({
                                    "task_id": task_idx,
                                    "score": task_score if task_passed else 0,
                                    "cases": cases_results
                                })
                                
                            result_data = {
                                "status": final_status,
                                "score": total_score,
                                "tasks": tasks_results,
                                "compile_info": compile_info,
                                "time_used_sec": max([c['time'] for t in tasks_results for c in t['cases']] or [0]),
                                "memory_used_kb": max([c['memory'] for t in tasks_results for c in t['cases']] or [0]),
                                "message": f"Score: {total_score}"
                            }
                        else:
                            # Fallback for single test (e.g. test_local)
                            result_data = {"status": "XX", "message": "meta.json not found", "compile_info": compile_info}

                    # --- Static Analysis Integration ---
                    if job.use_static_analysis and job.static_analysis_config:
                        try:
                            logger.info(f"Running static analysis for {job.submission_id}...")
                            # Parse config string to rules list
                            # e.g. "forbid_loops,forbid_arrays" or "--forbid-loops --forbid-arrays"
                            # Simple parsing: split by space or comma
                            rules = []
                            if "," in job.static_analysis_config:
                                rules = [r.strip() for r in job.static_analysis_config.split(",")]
                            else:
                                rules = job.static_analysis_config.split()
                            
                            # Ensure rules have -- prefix if missing (optional helper)
                            rules = [r if r.startswith("--") else f"--{r}" for r in rules]

                            api_key = os.getenv("SANDBOX_API_KEY", "default-insecure-key")
                            async with httpx.AsyncClient() as client:
                                resp = await client.post(
                                    "http://static-analysis:8000/scan",
                                    json={
                                        "code": source_code,
                                        "language": job.language,
                                        "rules": rules
                                    },
                                    headers={"X-API-KEY": api_key},
                                    timeout=10.0
                                )
                                if resp.status_code == 200:
                                    scan_result = resp.json()
                                    result_data["static_analysis"] = scan_result
                                    if not scan_result.get("success"):
                                        logger.warning(f"Static analysis failed: {scan_result.get('error')}")
                                    elif scan_result.get("violations"):
                                        # 靡態分析發現違規，設定狀態為 WA (Wrong Answer)
                                        violations = scan_result.get("violations", [])
                                        violation_messages = []
                                        for v in violations:
                                            rule = v.get("rule", "unknown")
                                            msg = v.get("message", "")
                                            line = v.get("line", 0)
                                            violation_messages.append(f"Line {line}: [{rule}] {msg}")
                                        
                                        result_data["status"] = "WA"
                                        result_data["score"] = 0
                                        result_data["message"] = f"Static Analysis Failed: {len(violations)} violation(s) found"
                                        result_data["static_analysis_violations"] = violation_messages
                                        logger.warning(f"Static analysis violations for {job.submission_id}: {violation_messages}")
                                else:
                                    logger.error(f"Static analysis service error: {resp.status_code} {resp.text}")
                                    result_data["static_analysis"] = {"error": f"Service Error {resp.status_code}"}
                        except Exception as e:
                            logger.error(f"Static analysis execution error: {e}")
                            result_data["static_analysis"] = {"error": str(e)}
                    # -----------------------------------

                    # Log Final Verdict (Removed duplicate log here)
                    # logger.info(f"Job {job.submission_id} Final Verdict: {result_data.get('status')} | Time: {result_data.get('time_used_sec')}s | Memory: {result_data.get('memory_used_kb')}KB")

                except Exception as e:
                    logger.error(f"Job execution error: {e}")
                    result_data = {"status": "XX", "message": str(e)}
                finally:
                    # Cleanup Proxy & Capture Logs
                    if job.allow_network:
                        if job.network_whitelist:
                            proxy_logs = await self.proxy.get_logs()
                            result_data["network_log"] = proxy_logs
                            await self.proxy.stop()
                        else:
                            result_data["network_log"] = "Full network access enabled (no proxy logs)"

                    # Cleanup Sidecar
                    if job.sidecar_image:
                        # Capture sidecar logs if something went wrong
                        if result_data.get("status") != "AC" and result_data.get("status") != "OK":
                            try:
                                if self.sidecar.container:
                                    sidecar_logs = self.sidecar.container.logs().decode('utf-8', errors='ignore')
                                    result_data["message"] += f"\n\n--- Sidecar Logs ---\n{sidecar_logs[-2000:]}" # Append last 2KB logs
                            except Exception as ex:
                                logger.warning(f"Failed to fetch sidecar logs: {ex}")
                        
                        await self.sidecar.stop()

                    # Cleanup uploaded files
                    try:
                        if job and job.file_path:
                            parent_dir = os.path.dirname(job.file_path)
                            # Security check: ensure we are deleting inside /tmp/sandbox/submissions
                            if os.path.exists(parent_dir) and "/tmp/sandbox/submissions" in parent_dir:
                                 shutil.rmtree(parent_dir, ignore_errors=True)
                                 logger.info(f"Cleaned up {parent_dir}")
                    except Exception as e:
                        logger.error(f"Cleanup error: {e}")

                # 3. 發送 Callback（根據提交類型選擇不同格式和 URL）
                if job.request_data.callback_url:
                    # 使用 Job 的 is_selftest 欄位判斷
                    is_selftest = getattr(job, 'is_selftest', False) or job.submission_id.startswith("selftest-")
                    
                    # 自動切換 callback URL
                    # 規格：正式提交 -> /submission/callback/
                    #       自定義測試 -> /submission/custom-test-callback/
                    callback_url = self._resolve_callback_url(job.request_data.callback_url, is_selftest)
                    
                    if is_selftest:
                        # 自定義測試 callback
                        callback_payload = self._build_selftest_callback_payload(job, result_data)
                    else:
                        # 正式提交 callback
                        callback_payload = self._build_submission_callback_payload(job, result_data)
                    
                    await self.send_callback(
                        callback_url, 
                        callback_payload,
                        token=job.request_data.callback_token,
                        is_selftest=is_selftest
                    )

                # 4. Record History
                # 保存原始請求資料（移除敏感資訊）
                original_request = None
                try:
                    req_dict = job.request_data.dict() if hasattr(job.request_data, 'dict') else vars(job.request_data)
                    # 移除過長的 code 欄位，保留其他資訊
                    original_request = {k: v for k, v in req_dict.items() if k != 'code'}
                    original_request['code_length'] = len(req_dict.get('code', '') or '')
                except Exception as e:
                    logger.warning(f"Failed to serialize original request: {e}")
                
                history_entry = {
                    "submission_id": job.submission_id,
                    "problem_id": job.problem_id,
                    "language": job.language,
                    "mode": job.mode,
                    "timestamp": job.timestamp,
                    "status": result_data.get("status"),
                    "time": result_data.get("time_used_sec"),
                    "memory": result_data.get("memory_used_kb"),
                    "message": result_data.get("message"),
                    "stdout": result_data.get("stdout", "")[:1000], # Keep first 1000 chars
                    "stderr": result_data.get("stderr", "")[:1000],
                    "network_log": result_data.get("network_log"),
                    "static_analysis": result_data.get("static_analysis"),
                    "tasks": result_data.get("tasks", []),
                    "compile_info": result_data.get("compile_info", {}),
                    "source_code": source_code[:50000] if source_code else None,  # Keep up to 50KB of source code
                    "original_request": original_request  # 原始 POST 請求資料
                }
                self.manager.add_history(history_entry)
                
                # Log detailed result for debugging
                logger.info(f"Job {job.submission_id} Details: {json.dumps(history_entry, default=str)}")
                
                # Log Final Verdict (Updated keys)
                logger.info(f"Job {job.submission_id} Final Verdict: {result_data.get('status')} | Time: {result_data.get('time_used_sec')}s | Memory: {result_data.get('memory_used_kb')}KB")

            except Exception as e:
                logger.error(f"Worker-{self.worker_id} error: {e}", exc_info=True)
            finally:
                self.current_job = None
                self.is_busy = False

    def _resolve_callback_url(self, callback_url: str, is_selftest: bool) -> str:
        """
        根據提交類型自動解析正確的 callback URL
        
        規格文件：
        - 正式提交: {BACKEND_BASE_URL}/submission/callback/
        - 自定義測試: {BACKEND_BASE_URL}/submission/custom-test-callback/
        
        支援的輸入格式：
        1. 完整 URL (含路徑): 直接使用
        2. Base URL (不含路徑): 自動附加正確的 endpoint
        3. Base URL 以 /callback 結尾: 替換為正確的 endpoint
        """
        from urllib.parse import urlparse, urlunparse
        
        parsed = urlparse(callback_url)
        path = parsed.path.rstrip('/')
        
        # 定義 endpoint 路徑
        SUBMISSION_CALLBACK = "/submission/callback"
        CUSTOM_TEST_CALLBACK = "/submission/custom-test-callback"
        
        # 判斷是否需要自動切換
        if path.endswith("/submission/callback") or path.endswith("/submission/custom-test-callback"):
            # 已有完整路徑，根據 is_selftest 切換
            base_path = path.rsplit("/submission/", 1)[0]
            new_path = base_path + (CUSTOM_TEST_CALLBACK if is_selftest else SUBMISSION_CALLBACK)
        elif "/submission/" in path:
            # 部分路徑，保持原樣
            new_path = path
        else:
            # 只有 base URL，附加正確的 endpoint
            new_path = path + (CUSTOM_TEST_CALLBACK if is_selftest else SUBMISSION_CALLBACK)
        
        # 確保路徑以 / 結尾（符合 Django 風格）
        if not new_path.endswith('/'):
            new_path += '/'
        
        resolved_url = urlunparse((
            parsed.scheme,
            parsed.netloc,
            new_path,
            parsed.params,
            parsed.query,
            parsed.fragment
        ))
        
        logger.debug(f"Resolved callback URL: {callback_url} -> {resolved_url} (is_selftest={is_selftest})")
        return resolved_url

    def _convert_status_to_backend_format(self, internal_status: str) -> str:
        """
        將內部狀態碼轉換為後端期望的格式
        內部: AC, WA, CE, TO, RE, XX, SG, MLE 等
        後端: accepted, wrong_answer, compile_error, time_limit_exceeded, memory_limit_exceeded, runtime_error
        """
        status_map = {
            "AC": "accepted",
            "WA": "wrong_answer",
            "CE": "compile_error",
            "TO": "time_limit_exceeded",
            "TLE": "time_limit_exceeded",
            "MLE": "memory_limit_exceeded",
            "RE": "runtime_error",
            "SG": "runtime_error",  # Signal killed -> runtime error
            "XX": "runtime_error",  # Internal error -> runtime error
            "OK": "accepted",       # OK without further check means accepted
        }
        return status_map.get(internal_status, "runtime_error")

    def _build_submission_callback_payload(self, job: Job, result_data: Dict[str, Any]) -> Dict[str, Any]:
        """
        構建正式提交的 callback payload，符合後端 API 規格
        """
        # 狀態轉換
        status = self._convert_status_to_backend_format(result_data.get("status", "XX"))
        
        # 計算總分 (0-100)
        score = result_data.get("score", 0)
        
        # 計算總執行時間和記憶體 (轉換為毫秒和 KB)
        time_sec = result_data.get("time_used_sec", 0) or 0
        execution_time = int(time_sec * 1000)  # 轉換為毫秒
        memory_usage = result_data.get("memory_used_kb", 0) or 0
        
        # 構建 test_results 陣列
        test_results = []
        
        tasks = result_data.get("tasks", [])
        for task in tasks:
            cases = task.get("cases", [])
            task_score = task.get("score", 0)
            task_id = task.get("task_id", 1)  # 子題 ID
            case_count = len(cases)
            score_per_case = task_score / case_count if case_count > 0 else 0
            case_index_in_task = 0  # 每個子題內的 index 從 0 開始
            
            for case in cases:
                case_index_in_task += 1
                case_status = self._convert_status_to_backend_format(case.get("status", "XX"))
                case_time = case.get("time", 0) or 0
                case_memory = case.get("memory", 0) or 0
                
                # 計算單個測試案例的得分
                case_passed = case.get("status") == "AC"
                case_score = int(score_per_case) if case_passed else 0
                max_score = int(score_per_case) if score_per_case > 0 else 10  # 預設每個 case 10 分
                
                test_results.append({
                    "test_case_id": task_id,  # 子題 ID (同一子題內的 case 共用此 ID)
                    "test_case_index": case_index_in_task,  # 子題內流水號，每個子題從 1 開始
                    "status": case_status,
                    "execution_time": int(case_time * 1000),  # 轉換為毫秒
                    "memory_usage": int(case_memory),
                    "score": case_score,
                    "max_score": max_score,
                    "error_message": case.get("message") if case.get("status") != "AC" else None
                })
        
        # 如果沒有 tasks（如編譯錯誤），也需要回傳基本資訊
        if not test_results and status == "compile_error":
            compile_info = result_data.get("compile_info", {})
            test_results.append({
                "test_case_id": 1,
                "test_case_index": 1,
                "status": "compile_error",
                "execution_time": 0,
                "memory_usage": 0,
                "score": 0,
                "max_score": 100,
                "error_message": compile_info.get("stderr", "") or compile_info.get("message", "Compilation failed")
            })
        
        # 如果靜態分析違規，也需要回傳錯誤資訊
        if not test_results and result_data.get("static_analysis_violations"):
            violations = result_data.get("static_analysis_violations", [])
            violation_detail = "\n".join(violations) if violations else "Static analysis violation detected"
            test_results.append({
                "test_case_id": 1,
                "test_case_index": 1,
                "status": "wrong_answer",
                "execution_time": 0,
                "memory_usage": 0,
                "score": 0,
                "max_score": 100,
                "error_message": f"靜態分析違規:\n{violation_detail}"
            })
        
        # 靜態分析結果 (如果有)
        static_analysis = result_data.get("static_analysis")
        
        # 網路日誌 (如果有)
        network_log = result_data.get("network_log")
        
        payload = {
            "submission_id": job.submission_id,
            "status": status,
            "score": score,
            "execution_time": execution_time,
            "memory_usage": memory_usage,
            "test_results": test_results
        }
        
        if static_analysis:
            payload["static_analysis"] = static_analysis
        
        if network_log:
            payload["network_log"] = network_log
        
        return payload

    def _build_selftest_callback_payload(self, job: Job, result_data: Dict[str, Any]) -> Dict[str, Any]:
        """
        構建自定義測試的 callback payload，符合後端 API 規格
        """
        # 狀態：completed 或 error
        internal_status = result_data.get("status", "XX")
        status = "completed" if internal_status in ["AC", "OK", "WA"] else "error"
        
        # 計算執行時間和記憶體 (轉換為毫秒和 KB)
        time_sec = result_data.get("time_used_sec", 0) or 0
        execution_time = int(time_sec * 1000)
        memory_usage = result_data.get("memory_used_kb", 0) or 0
        
        # 取得輸出
        stdout = result_data.get("stdout", "")
        stderr = result_data.get("stderr", "")
        message = result_data.get("message", "")
        
        # 根據不同錯誤類型，補充 stderr 訊息
        # 靜態分析違規時，檢查 static_analysis_violations 欄位
        if result_data.get("static_analysis_violations"):
            # 靜態分析違規
            violations = result_data.get("static_analysis_violations", [])
            violation_detail = "\n".join(violations) if violations else "Static analysis violation detected"
            stderr = f"Static Analysis Failed (靜態分析違規):\n{violation_detail}"
            status = "error"  # 違規視為 error
        elif internal_status == "CE":
            # 編譯錯誤：stderr 放編譯訊息
            compile_info = result_data.get("compile_info", {})
            stderr = compile_info.get("stderr", "") or compile_info.get("message", "Compilation failed")
        elif internal_status in ["TO", "TLE"]:
            # 超時：補充超時訊息
            if not stderr:
                stderr = f"Time Limit Exceeded (執行超時). {message}".strip()
        elif internal_status == "MLE":
            # 記憶體超限
            if not stderr:
                stderr = f"Memory Limit Exceeded (記憶體超限). {message}".strip()
        elif internal_status in ["RE", "SG"]:
            # 執行時錯誤
            if not stderr:
                stderr = f"Runtime Error. {message}".strip()
        elif internal_status == "XX":
            # 內部錯誤
            if not stderr:
                stderr = f"Internal Error. {message}".strip()
        
        # exit_code: 0 表示正常
        exit_code = 0 if internal_status in ["AC", "OK"] else 1
        
        # 靜態分析結果 (如果有)
        static_analysis = result_data.get("static_analysis")
        
        # 網路日誌 (如果有)
        network_log = result_data.get("network_log")
        
        payload = {
            "submission_id": job.submission_id,
            "status": status,
            "stdout": stdout,
            "stderr": stderr,
            "execution_time": execution_time,
            "memory_usage": memory_usage,
            "exit_code": exit_code
        }
        
        if static_analysis:
            payload["static_analysis"] = static_analysis
        
        if network_log:
            payload["network_log"] = network_log
        
        return payload

    async def send_callback(self, url: str, data: Dict[str, Any], token: str = None, retries: int = 3, is_selftest: bool = False):
        """發送 Webhook 回調 (帶重試)"""
        headers = {
            "Content-Type": "application/json"
        }
        # 使用 X-API-KEY header（根據後端文件規格）
        api_key = os.getenv("SANDBOX_API_KEY", "default-insecure-key")
        headers["X-API-KEY"] = api_key
        
        if token:
            headers["Authorization"] = f"Bearer {token}"
        
        # 指數退避重試間隔
        backoff_intervals = [1, 2, 4]
            
        async with httpx.AsyncClient() as client:
            for attempt in range(retries):
                try:
                    resp = await client.post(url, json=data, headers=headers, timeout=30.0)
                    if resp.status_code == 200:
                        logger.info(f"Callback sent successfully to {url} (Attempt {attempt+1})")
                        return True
                    elif resp.status_code >= 500:
                        # 5xx 錯誤需要重試
                        logger.warning(f"Callback failed with {resp.status_code}, will retry (Attempt {attempt+1})")
                    else:
                        # 4xx 錯誤不重試
                        logger.error(f"Callback failed with {resp.status_code}: {resp.text} (Attempt {attempt+1})")
                        return False
                except httpx.TimeoutException:
                    logger.warning(f"Callback timeout (Attempt {attempt+1})")
                except Exception as e:
                    logger.warning(f"Callback error: {e} (Attempt {attempt+1})")
                
                # 指數退避
                if attempt < retries - 1:
                    await asyncio.sleep(backoff_intervals[attempt])
        
        logger.error(f"Callback to {url} failed after {retries} attempts")
        return False

    async def stop(self):
        """優雅停止 Worker"""
        logger.info(f"Worker-{self.worker_id} stopping...")
        self.running = False

class WorkerManager:
    def __init__(self, num_workers: int = None):
        if num_workers is None:
            # Try to get from env var first
            env_workers = os.getenv("SANDBOX_WORKERS")
            if env_workers:
                env_workers = env_workers.strip()
                if env_workers.isdigit():
                    num_workers = int(env_workers)
                    logger.info(f"Configured {num_workers} workers from SANDBOX_WORKERS environment variable.")
                else:
                    logger.warning(f"Invalid SANDBOX_WORKERS value: '{env_workers}'. Using default.")
            
            if num_workers is None:
                num_workers = os.cpu_count() or 4 # Default to number of CPU cores
                logger.info(f"Configured {num_workers} workers based on CPU count (default).")
        else:
            logger.info(f"Configured {num_workers} workers from argument.")
        
        self.num_workers = num_workers
        self.workers: List[Worker] = []
        self.tasks: List[asyncio.Task] = []
        self.history: List[Dict[str, Any]] = [] # Simple in-memory history
        
    async def start(self):
        """啟動所有 Workers"""
        logger.info(f"Starting {self.num_workers} workers...")
        for i in range(self.num_workers):
            worker = Worker(worker_id=i, queue=job_queue, manager=self)
            self.workers.append(worker)
            self.tasks.append(asyncio.create_task(worker.run()))
            
    async def stop(self):
        """停止所有 Workers 並等待完成"""
        logger.info("Stopping workers...")
        for worker in self.workers:
            await worker.stop()
        if self.tasks:
            await asyncio.gather(*self.tasks, return_exceptions=True)
        logger.info("All workers stopped.")

    def add_history(self, job_data: Dict[str, Any]):
        """加入歷史紀錄 (保留最近 50 筆) 並寫入檔案"""
        self.history.insert(0, job_data)
        if len(self.history) > 50:
            self.history.pop()

        # Persist to file
        try:
            log_path = os.path.join("logs", "submission_history.jsonl")
            # Ensure directory exists
            os.makedirs(os.path.dirname(log_path), exist_ok=True)
            with open(log_path, "a", encoding="utf-8") as f:
                f.write(json.dumps(job_data) + "\n")
        except Exception as e:
            logger.error(f"Failed to write history log: {e}")

    def get_status(self) -> Dict[str, Any]:
        """獲取 Worker 狀態"""
        active_count = sum(1 for w in self.workers if w.is_busy)
        return {
            "total_workers": self.num_workers,
            "active_workers": active_count,
            "idle_workers": self.num_workers - active_count,
            "workers_details": [
                {
                    "id": w.worker_id, 
                    "busy": w.is_busy, 
                    "current_job": w.current_job.submission_id if w.current_job else None
                }
                for w in self.workers
            ],
            "history": self.history
        }

# 全域單例
worker_manager = WorkerManager()