diff --git a/.env.example b/.env.example index f49466fe..b7f21bb9 100644 --- a/.env.example +++ b/.env.example @@ -73,6 +73,7 @@ ENABLE_REAL_LLM_CALL=true # optional | LLM 실제 호출 DYNAMIC_ANALYSIS_TIMEOUT_SEC=120 # optional | 전체 동적 분석 타임아웃(초) DYNAMIC_HARNESS_CLOSE_TIMEOUT_SEC=10 # optional | Playwright 종료 대기 타임아웃(초) DYNAMIC_HARNESS_HEADLESS=true # optional | Playwright headless 모드 여부 +DYNAMIC_MOCK_AUTOSTART=true # optional | Start local mock page server automatically SERVICE_WORKER_TIMEOUT_MS=5000 # optional | 서비스 워커 등록 대기 타임아웃(ms) DISPLAY=:99 # optional | headless 환경 가상 디스플레이 (Linux) LLM_DEBUG_PROMPT_DUMP_DIR= # optional | LLM 프롬프트 덤프 디렉토리 (디버깅용) diff --git a/ExtAnalysis/reports.json b/ExtAnalysis/reports.json index cb5df5e6..8de08ae1 100644 --- a/ExtAnalysis/reports.json +++ b/ExtAnalysis/reports.json @@ -1175,6 +1175,62 @@ "report_directory": "/EXA2026156043442", "time": "2026-06-05 04:34:42", "version": "3.2.2" + }, + { + "id": "EXA2026160091302", + "name": "ResuMatch - Free Offline Keyword Analyzer", + "report_directory": "\\EXA2026160091302", + "time": "2026-06-09 09:13:02", + "version": "1.0.0" + }, + { + "id": "EXA2026161091908", + "name": "Python Assist", + "report_directory": "\\EXA2026161091908", + "time": "2026-06-10 09:19:08", + "version": "1.1" + }, + { + "id": "EXA2026161091945", + "name": "Python Tutor - Wiingy", + "report_directory": "\\EXA2026161091945", + "time": "2026-06-10 09:19:46", + "version": "1.1" + }, + { + "id": "EXA2026161092038", + "name": "Selenium Auto Code Generator (Python)", + "report_directory": "\\EXA2026161092038", + "time": "2026-06-10 09:20:40", + "version": "2.1" + }, + { + "id": "EXA2026161092200", + "name": "__MSG_appName__", + "report_directory": "\\EXA2026161092200", + "time": "2026-06-10 09:22:01", + "version": "1.0" + }, + { + "id": "EXA2026161144101", + "name": "\uc5c5\ubb34\ud3ec\ud138(+EVPN) \ub85c\uadf8\uc778 \uc720\uc9c0 \ub3c4\uc6b0\ubbf8", + "report_directory": "\\EXA2026161144101", + "time": "2026-06-10 14:41:01", + "version": "1.3.2" + }, + { + "id": "EXA2026161144522", + "name": "__MSG_appName__", + "report_directory": "\\EXA2026161144522", + "time": "2026-06-10 14:45:22", + "version": "2.0.0" + }, + { + "id": "EXA2026161145217", + "name": "__MSG_appName__", + "report_directory": "\\EXA2026161145217", + "time": "2026-06-10 14:52:18", + "version": "5.7" } ] } \ No newline at end of file diff --git a/backend/risk_scoring.py b/backend/risk_scoring.py index d2de662a..a5855be0 100644 --- a/backend/risk_scoring.py +++ b/backend/risk_scoring.py @@ -348,26 +348,22 @@ def calculate_weighted_final_risk( if risk_level in {"HIGH", "CRITICAL"}: blockers.append(f"final weighted risk level is {risk_level}") - # decision: only LOW is auto-approved; every non-safe outcome goes to review. - if risk_level == "LOW": - recommended = "approve" - else: - recommended = "review" + # Suppressor only reports scan risk. ExtS3 applies operational approval policy. + recommended = "review" if dynamic["status"] in {"error", "skipped"} and ( static["risk_level"] in {"MEDIUM", "HIGH", "CRITICAL"} or obf["risk_level"] in {"MEDIUM", "HIGH", "CRITICAL"} ): - if recommended == "approve": - recommended = "review" + recommended = "review" - if error_or_skipped >= 2 and recommended == "approve": + if error_or_skipped >= 2: recommended = "review" - if recommended == "review": - decision_reason = "Risk signals or analysis uncertainty require human review before approval." + if risk_level == "LOW": + decision_reason = "No strong dynamic or corroborated static/obfuscation risk indicators were detected; ExtS3 policy determines final approval." else: - decision_reason = "No strong dynamic or corroborated static/obfuscation risk indicators were detected." + decision_reason = "Risk signals or analysis uncertainty require human review before approval." return { "risk_level": risk_level, diff --git a/backend/web_payload.py b/backend/web_payload.py index 41021253..0b84d98c 100644 --- a/backend/web_payload.py +++ b/backend/web_payload.py @@ -309,7 +309,7 @@ def summarize_rag_result(rag_fingerprint_result: dict, rag_rerank_result: dict) def infer_recommended_decision(payload: dict) -> str: overall = _safe_dict(payload.get("overall")) weighted_decision = str(overall.get("recommended_decision", "")).strip().lower() - if weighted_decision in {"approve", "review", "reject"}: + if weighted_decision in {"review", "reject"}: return weighted_decision overall = _safe_dict(payload.get("overall")) @@ -333,7 +333,7 @@ def infer_recommended_decision(payload: dict) -> str: if risk == "UNKNOWN" or error_count >= 2: return "review" - return "approve" + return "review" def _build_review_fields(payload: dict) -> tuple[list[str], list[str], list[str]]: @@ -467,7 +467,7 @@ def build_web_payload( "overall": { "risk_level": overall_level, "risk_score": overall_score, - "recommended_decision": weighted_decision if weighted_decision in {"approve", "review"} else "review", + "recommended_decision": "review", "decision_reason": weighted_reason, "summary": _clip_text( _safe_dict(dynamic_result.get("final_risk") if isinstance(dynamic_result, dict) else {}).get("reason") @@ -504,20 +504,15 @@ def build_web_payload( final_decision = str(decision).strip().lower() if decision else inferred if final_decision == "reject": final_decision = "review" - if final_decision not in {"approve", "review"}: - final_decision = inferred - if final_decision == "reject": + if final_decision != "review": final_decision = "review" payload["overall"]["recommended_decision"] = final_decision if not payload["overall"].get("decision_reason"): - payload["overall"]["decision_reason"] = { - "review": "Some risk indicators or analysis uncertainty require human validation.", - "approve": "No significant risk indicators were detected in the summarized analyses.", - }[final_decision] + payload["overall"]["decision_reason"] = "ExtS3 policy determines final approval from the scan risk result." review_reasons, blockers, actions = _build_review_fields(payload) - payload["review"]["needs_human_review"] = final_decision != "approve" + payload["review"]["needs_human_review"] = True payload["review"]["review_reasons"] = review_reasons payload["review"]["approval_blockers"] = blockers payload["review"]["recommended_actions"] = actions diff --git a/docker-compose.pgvector.yml b/docker-compose.pgvector.yml new file mode 100644 index 00000000..41869451 --- /dev/null +++ b/docker-compose.pgvector.yml @@ -0,0 +1,20 @@ +services: + vector-db: + image: pgvector/pgvector:pg16 + container_name: suppressor-vector-db + environment: + POSTGRES_USER: "${PGVECTOR_DB_USER:-vector_db_user}" + POSTGRES_PASSWORD: "${PGVECTOR_DB_PASSWORD:-vector_db_password}" + POSTGRES_DB: "${PGVECTOR_DB_NAME:-vector_db}" + ports: + - "${PGVECTOR_DB_PUBLISHED_PORT:-5433}:5432" + volumes: + - suppressor_vector_db_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${PGVECTOR_DB_USER:-vector_db_user} -d ${PGVECTOR_DB_NAME:-vector_db}"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + suppressor_vector_db_data: diff --git a/embedding/base_db.py b/embedding/base_db.py index 1b1475da..aff1c3f9 100644 --- a/embedding/base_db.py +++ b/embedding/base_db.py @@ -1,31 +1,18 @@ -import os import json +import os from pathlib import Path from typing import Any -from dotenv import load_dotenv import requests -from supabase import create_client, Client - -load_dotenv() - -# 1. Supabase 설정 -url: str | None = os.environ.get("SUPABASE_URL") -key: str | None = os.environ.get("DB_KEY") - -if not url: - raise RuntimeError("SUPABASE_URL 환경변수가 없습니다.") +from dotenv import load_dotenv -if not key: - raise RuntimeError("DB_KEY 환경변수가 없습니다.") +from embedding.embed import embed_fingerprint +from embedding.pgvector_store import count_vectors, insert_vector_record -supabase: Client = create_client(url, key) +load_dotenv() def embed_full_text(text: str) -> list[float]: - """ - Ollama bge-m3 모델을 사용하여 텍스트 임베딩 추출 - """ embed_url = os.environ.get("OLLAMA_EMBED_URL", "http://localhost:11434/api/embed") legacy_embed_url = os.environ.get("OLLAMA_EMBED_LEGACY_URL", "http://localhost:11434/api/embeddings") model = os.environ.get("EMBEDDING_MODEL", "bge-m3") @@ -58,74 +45,38 @@ def embed_full_text(text: str) -> list[float]: if isinstance(legacy_vector, list): return legacy_vector - raise RuntimeError("지원하지 않는 Ollama embedding 응답 타입") - - except Exception as e: - print(f"❌ 임베딩 추출 중 오류 발생: {e}") + raise RuntimeError("unsupported Ollama embedding response") + except Exception as exc: + print(f"[pgvector] embedding failed: {exc}") return [] def load_base_json(file_path: Path) -> dict[str, Any]: - """ - base/*.json 파일 로드 - 지원 형식: - - 1. wrapper 형식: - { - "pattern_name": "...", - "doc_ref": "scenario_docs/....md", - "vector_fingerprint": {...} - } - - 2. vector_fingerprint만 들어있는 형식: - { - "manifest_profile": {...}, - "capability_profile": [...], - ... - } - """ with file_path.open("r", encoding="utf-8") as f: data = json.load(f) - if not isinstance(data, dict): - raise ValueError("JSON 최상위 구조는 object(dict)여야 합니다.") - + raise ValueError("base JSON root must be an object") return data def normalize_base_record(file_path: Path, data: dict[str, Any]) -> dict[str, Any]: - """ - base JSON을 Vector DB 저장용 표준 payload로 변환한다. - - 반환 형식: - { - "pattern_name": "...", - "doc_ref": "scenario_docs/....md", - "vector_fingerprint": {...} - } - """ file_stem = file_path.stem - # wrapper 형식이면 vector_fingerprint 필드를 사용 if isinstance(data.get("vector_fingerprint"), dict): pattern_name = data.get("pattern_name") or file_stem doc_ref = data.get("doc_ref") or f"scenario_docs/{pattern_name}.md" vector_fingerprint = data["vector_fingerprint"] - - # raw fingerprint 형식이면 파일명 기반으로 wrapper 생성 else: pattern_name = data.get("pattern_name") or file_stem doc_ref = data.get("doc_ref") or f"scenario_docs/{pattern_name}.md" vector_fingerprint = data if not isinstance(pattern_name, str) or not pattern_name.strip(): - raise ValueError("pattern_name이 비어 있습니다.") - + raise ValueError("pattern_name is empty") if not isinstance(doc_ref, str) or not doc_ref.strip(): - raise ValueError("doc_ref가 비어 있습니다.") - + raise ValueError("doc_ref is empty") if not isinstance(vector_fingerprint, dict) or not vector_fingerprint: - raise ValueError("vector_fingerprint가 비어 있거나 dict가 아닙니다.") + raise ValueError("vector_fingerprint is empty or invalid") required_keys = [ "manifest_profile", @@ -134,10 +85,9 @@ def normalize_base_record(file_path: Path, data: dict[str, Any]) -> dict[str, An "predicted_flows", "behavior_tags", ] - missing = [key for key in required_keys if key not in vector_fingerprint] if missing: - raise ValueError(f"vector_fingerprint 필수 키 누락: {missing}") + raise ValueError(f"vector_fingerprint missing required keys: {missing}") return { "pattern_name": pattern_name, @@ -147,11 +97,6 @@ def normalize_base_record(file_path: Path, data: dict[str, Any]) -> dict[str, An def build_embedding_text(vector_fingerprint: dict[str, Any]) -> str: - """ - 실제 임베딩에 사용할 텍스트. - pattern_name/doc_ref까지 섞으면 유사도에 불필요한 영향을 줄 수 있으므로 - vector_fingerprint만 임베딩한다. - """ return json.dumps( vector_fingerprint, ensure_ascii=False, @@ -161,128 +106,77 @@ def build_embedding_text(vector_fingerprint: dict[str, Any]) -> str: def build_document_payload(record: dict[str, Any]) -> str: - """ - DB document 컬럼에 저장할 JSON 문자열. - - compareDB/rerank 단계에서 이 값을 json.loads해서 - payload.pattern_name, payload.doc_ref, payload.vector_fingerprint로 복구한다. - """ payload = { "pattern_name": record["pattern_name"], "doc_ref": record["doc_ref"], "vector_fingerprint": record["vector_fingerprint"], } - - return json.dumps( - payload, - ensure_ascii=False, - sort_keys=True, - ) + return json.dumps(payload, ensure_ascii=False, sort_keys=True) def validate_doc_ref_exists(project_root: Path, doc_ref: str) -> bool: - """ - scenario_docs 문서가 실제 존재하는지 확인. - 없다고 저장을 막지는 않고 경고만 출력한다. - """ - doc_path = project_root / doc_ref - return doc_path.exists() + return (project_root / doc_ref).exists() def store_all_knowledge_base() -> None: project_root = Path(__file__).resolve().parent - - # 이 스크립트가 embedding/ 안에 있고 base도 embedding/base라면 이 경로가 맞음 base_dir = project_root / "base" - - # 만약 base가 프로젝트 루트/base에 있다면 fallback if not base_dir.exists(): alt_base_dir = project_root.parent / "base" if alt_base_dir.exists(): base_dir = alt_base_dir if not base_dir.exists(): - print(f"❌ base 폴더를 찾을 수 없습니다: {base_dir}") + print(f"[pgvector] base directory not found: {base_dir}") return json_files = sorted(base_dir.glob("*.json")) - if not json_files: - print("💡 처리할 JSON 파일이 없습니다.") + print("[pgvector] no base JSON files found") return - print(f"🚀 총 {len(json_files)}개의 base JSON 처리를 시작합니다.") - print(f"📁 base_dir = {base_dir}") - + print(f"[pgvector] seeding {len(json_files)} base scenario embeddings from {base_dir}") success_count = 0 fail_count = 0 for index, file_path in enumerate(json_files, start=1): - print("\n" + "=" * 80) - print(f"🔄 [{index}/{len(json_files)}] {file_path.name} 처리 시작") - try: raw_data = load_base_json(file_path) record = normalize_base_record(file_path, raw_data) - - pattern_name = record["pattern_name"] doc_ref = record["doc_ref"] vector_fingerprint = record["vector_fingerprint"] - print(f" pattern_name: {pattern_name}") - print(f" doc_ref: {doc_ref}") - print(f" vector_fingerprint keys: {list(vector_fingerprint.keys())}") - if not validate_doc_ref_exists(project_root.parent, doc_ref) and not validate_doc_ref_exists(project_root, doc_ref): - print(f" ⚠️ scenario doc 파일을 찾지 못했습니다: {doc_ref}") - print(" ⚠️ DB 저장은 계속하지만, Dynamic RAG 실행 시 doc_ref 로드가 실패할 수 있습니다.") + print(f"[pgvector] warning: scenario doc not found for {record['pattern_name']}: {doc_ref}") - # 1. 임베딩은 vector_fingerprint만 사용 - embedding_text = build_embedding_text(vector_fingerprint) + vector = embed_fingerprint(vector_fingerprint) + if not vector: + raise RuntimeError("embedding vector generation failed") - print(" 🧠 임베딩 생성 중...") - vector = embed_full_text(embedding_text) + document_payload = build_document_payload(record) + insert_vector_record(document_payload, vector) + success_count += 1 + print(f"[pgvector] seeded {index}/{len(json_files)}: {record['pattern_name']} dim={len(vector)}") + except Exception as exc: + fail_count += 1 + print(f"[pgvector] failed to seed {file_path.name}: {exc}") - if not vector: - raise RuntimeError("embedding vector 생성 실패") + print(f"[pgvector] seed complete: success={success_count} fail={fail_count}") - print(f" ✅ 임베딩 생성 완료: dim={len(vector)}") - # 2. document에는 pattern_name/doc_ref/vector_fingerprint wrapper JSON 저장 - document_payload = build_document_payload(record) +def ensure_knowledge_base_seeded() -> None: + try: + existing_count = count_vectors() + except Exception as exc: + print(f"[pgvector] knowledge base check failed: {exc}") + return - insert_data = { - "document": document_payload, - "embedding": vector, - } - - # 현재 테이블 구조가 document, embedding만 있다고 가정 - # pattern_name/doc_ref 컬럼이 따로 있다면 아래 주석을 해제하고 테이블 컬럼도 맞춰야 함 - # - # insert_data = { - # "document": document_payload, - # "pattern_name": pattern_name, - # "doc_ref": doc_ref, - # "embedding": vector, - # } - - try: - response = supabase.table("es3_vector").insert(insert_data).execute() - print(f" ✅ DB 저장 완료: {pattern_name}") - success_count += 1 - - except Exception as db_e: - print(f" ❌ DB 저장 중 오류 발생: {db_e}") - fail_count += 1 - - except Exception as e: - print(f"❌ {file_path.name} 처리 중 오류 발생: {e}") - fail_count += 1 + if existing_count > 0: + print(f"[pgvector] knowledge base already loaded: {existing_count} vectors") + return - print("\n" + "=" * 80) - print("📌 Vector DB seed 완료") - print(f"✅ 성공: {success_count}") - print(f"❌ 실패: {fail_count}") + print("[pgvector] knowledge base is empty; creating embeddings from embedding/base") + store_all_knowledge_base() if __name__ == "__main__": diff --git a/embedding/compare.py b/embedding/compare.py index c598f9a8..9d62faaa 100644 --- a/embedding/compare.py +++ b/embedding/compare.py @@ -1,7 +1,10 @@ import json import os + from dotenv import load_dotenv +from embedding.pgvector_store import search_vectors + load_dotenv() FALLBACK_PATTERN_NAME = "session_storage_exfiltration_document_start" @@ -47,13 +50,11 @@ def _unwrap_document_payload(document_obj, row): elif _is_probably_fingerprint(document_obj): vf = document_obj - # nested wrapper: {"vector_fingerprint": {"pattern_name":..,"doc_ref":..,"vector_fingerprint": {...}}} if isinstance(vf, dict) and "vector_fingerprint" in vf: pattern_name = vf.get("pattern_name") or pattern_name doc_ref = vf.get("doc_ref") or doc_ref vf = vf.get("vector_fingerprint") - # if still wrapper-like, keep unwrapping once more defensively if isinstance(vf, dict) and "vector_fingerprint" in vf: vf = vf.get("vector_fingerprint") @@ -61,18 +62,11 @@ def _unwrap_document_payload(document_obj, row): vf = None if not isinstance(pattern_name, str) or not pattern_name.strip(): - # row id가 문자열이면 fallback name으로 사용 rid = row.get("id") - if isinstance(rid, str) and rid.strip(): - pattern_name = rid.strip() - else: - pattern_name = FALLBACK_PATTERN_NAME + pattern_name = rid.strip() if isinstance(rid, str) and rid.strip() else FALLBACK_PATTERN_NAME if not isinstance(doc_ref, str) or not doc_ref.strip(): - if pattern_name and pattern_name != FALLBACK_PATTERN_NAME: - doc_ref = f"scenario_docs/{pattern_name}.md" - else: - doc_ref = FALLBACK_DOC_REF + doc_ref = f"scenario_docs/{pattern_name}.md" if pattern_name != FALLBACK_PATTERN_NAME else FALLBACK_DOC_REF return pattern_name, doc_ref, vf @@ -122,62 +116,25 @@ def normalize_compare_result_rows(results: list[dict]) -> list[dict]: def compareDB(embedding_vector: list[float]): - """ - 임베딩 벡터를 받아 DB와 비교하고, 결과를 터미널에 출력한 뒤 데이터를 반환합니다. - """ - url: str = os.environ.get("SUPABASE_URL") - key: str = os.environ.get("DB_KEY") - - if not url or not key: - print("❌ Supabase 설정이 누락되었습니다.") - return [] - try: - from supabase import create_client - except Exception: - print("❌ supabase 패키지가 설치되어 있지 않습니다.") - return [] - - supabase = create_client(url, key) - - try: - # 1. Supabase RPC 호출 (모든 유사도를 보기 위해 threshold=0 설정) - response = supabase.rpc( - 'search_fingerprints', - { - 'query_embedding': embedding_vector, - 'match_threshold': 0, - 'match_count': 10 # 상위 10개 정도 출력 - } - ).execute() - - results = response.data if response.data else [] + results = search_vectors( + embedding_vector, + match_threshold=float(os.environ.get("PGVECTOR_MATCH_THRESHOLD", "0")), + match_count=int(os.environ.get("PGVECTOR_MATCH_COUNT", "10")), + ) normalized_results = normalize_compare_result_rows(results) - # 2. 유사도 리포트 출력 로직 - print("\n" + "="*65) - print(f"📊 보안 분석 유사도 리포트 (검색된 지식 베이스: {len(normalized_results)}건)") + print("\n" + "=" * 65) + print(f"[pgvector] Vector similarity search result: {len(normalized_results)} rows") print("-" * 65) - print(f"{'순위':<4} | {'상태':<2} | {'유사도 점수':<10} | {'데이터 ID (UUID)'}") + print(f"{'rank':<4} | {'score':<10} | {'id'}") print("-" * 65) + for i, res in enumerate(normalized_results, start=1): + score_pct = float(res.get("score", 0.0)) * 100 + print(f"{i:2d} | {score_pct:8.2f}% | {res.get('id')}") + print("=" * 65 + "\n") - if not normalized_results: - print(" 매칭되는 데이터가 DB에 존재하지 않습니다.") - else: - for i, res in enumerate(normalized_results): - score_pct = float(res.get("score", 0.0)) * 100 - match_id = res.get("id") - - # 유사도에 따른 상태 아이콘 설정 - status = "🔥" if score_pct > 80 else "🔍" if score_pct > 50 else "❔" - - print(f"{i+1:2d} | {status} | {score_pct:8.2f}% | {match_id}") - - print("="*65 + "\n") - - # 3. rerank 친화 정규화 결과 반환 return normalized_results - - except Exception as e: - print(f"❌ DB 비교 중 오류 발생: {e}") + except Exception as exc: + print(f"[pgvector] vector search failed: {exc}") return [] diff --git a/embedding/embedding.json b/embedding/embedding.json index f5d20ae7..ed9b8403 100644 --- a/embedding/embedding.json +++ b/embedding/embedding.json @@ -1,1026 +1,1026 @@ [ - -0.03324944, - -0.0050444356, - -0.022208717, - -0.013840821, - -0.036306843, - -0.0022242917, - 0.026391977, - 0.042421598, - -0.00089923106, - 0.009452817, - -0.031261306, - -0.005864351, - -0.014947828, - -0.006703557, - 0.004853893, - -0.005408846, - 0.013212153, - 0.014529799, - -0.0064452803, - -0.008707598, - -0.025150225, - 0.0016150109, - 0.0034976674, - 0.017276108, - -0.038031578, - 0.028839972, - -0.013005501, - -0.06473733, - -0.02408848, - 0.03854639, - -0.0064084264, - -0.028682206, - 0.029870603, - 0.0059221857, - -0.027606811, - -0.030213438, - -0.012253738, - -0.02734291, - -0.08346272, - 0.0033594868, - -0.03274521, - 0.01222369, - 0.0026372166, - -0.028184094, - 0.0062350375, - -0.03290036, - 0.0021557559, - -0.025799992, - -0.013827796, - -0.059359822, - -0.027723094, - -0.007625635, - 0.005624747, - -0.016466785, - 0.041931044, - 0.03902451, - -0.03521946, - -0.005502113, - -0.051538147, - 0.014607151, - -0.057202943, - -0.010779941, - -0.020906843, - 0.00303758, - 0.0054768804, - 0.025793593, - 0.021582622, - -0.0006367268, - -0.006949201, - -0.032705393, - -0.00420725, - 0.017624665, - -0.022187153, - -0.010142849, - -0.061786655, - 0.03516763, - 0.0035869377, - -0.035675116, - -0.010795566, - 0.023055978, - -0.03176888, - -0.0012049475, - 0.04511863, - 0.00087903417, - -0.009443859, - 0.032468427, - -0.011862333, - 0.03334763, - 0.050263852, - 0.0062432084, - -0.012368068, - -0.028739965, - 0.060837105, - -0.04360381, - -0.035042245, - 0.004143182, - -0.0030015605, - -0.02348868, - 0.046387035, - 0.006537461, - -0.009764383, - 0.016632792, - 0.0075989203, - -0.0059683276, - 0.025636151, - 0.043208137, - 0.018790979, - -0.002043801, - 0.012660172, - -0.012858403, - -0.015034742, - 0.024507359, - 0.036754955, - 0.028024843, - -0.01150434, - -0.045861814, - 0.010035876, - -0.021241153, - -0.026187846, - -0.01570518, - 0.02864863, - 0.05583761, - 0.03970998, - -0.029948315, - 0.037919667, - 0.011180024, - 0.031077484, - 0.023687935, - -0.006760123, - 0.03169908, - 0.020260377, - 0.033927288, - -0.004681003, - 0.0014895055, - -0.0734741, - -0.010337047, - 0.0081940405, - 0.04981689, - 0.04210615, - -0.040272374, - 0.017698646, - 0.022419874, - -0.03152284, - -0.019055769, - 0.023887446, - -0.07555047, - 0.023879698, - 0.049177237, - -0.010008643, - -0.062070493, - 0.03615153, - 0.009455564, - 0.051024348, - 0.05038881, - -0.023731126, - -0.076091565, - 0.00976865, - 0.0027058173, - 0.05945507, - -0.0035481045, - -0.0199888, - 0.014231154, - -0.036088984, - 0.04085355, - 0.034093622, - -0.00878334, - 0.0027799108, - -0.01854704, - -0.043396227, - -0.0087719755, - 0.018955313, - -0.04134611, - -0.0025690554, - 0.021780485, - 0.0038092167, - 0.002197573, - 0.06869432, - 0.026963985, - -0.008388767, - -0.02880179, - -0.055994406, - -0.004211152, - -0.019245846, - -0.02292355, - -0.019101774, - 0.033828583, - -0.02038171, - 0.00042381696, - -0.018327216, - 0.037831917, - 0.006654962, - -0.006109558, - 0.03155425, - 0.017479725, - -0.04884529, - -0.026817704, - 0.032447737, - -0.0017020075, - 0.014290424, - -0.02742174, - 0.022978669, - 0.03185352, - 0.04585362, - -0.021267036, - -0.037501656, - -0.018048106, - -0.0057161846, - -0.032997586, - -0.0017063415, - -0.0036430887, - 0.011584586, - 0.009970135, - -0.0024523325, - -0.0121574, - -0.028583424, - -0.020318493, - 0.00426731, - -0.0063978443, - 0.009322606, - -0.027304916, - 0.002994939, - 0.016257452, - 0.0054080305, - -0.0070847105, - -0.0029157556, - 0.010450153, - 0.025088038, - 0.008163413, - -0.055223655, - 0.022017531, - -0.013243919, - 0.03399588, - 0.022574188, - 0.00093703595, - 0.00882443, - -0.04569273, - 0.014744913, - 0.023944383, - 0.03197983, - -0.013947919, - 0.048221752, - -0.06261975, - 0.038866762, - -0.01505409, - -0.032155372, - -0.00022342797, - 0.003371724, - 0.054939497, - -0.04370387, - -0.026764233, - -0.018814465, - 0.01481644, - -0.042625267, - -0.006439748, - 0.02203547, - 0.0065768245, - 0.010242947, - 0.027784664, - -0.00065410376, - 0.02007811, - -0.010030368, - 0.0128436405, - 0.020202577, - 0.02748119, - 0.028105304, - -0.008017908, - -0.0063449796, - -0.021282565, - 0.004351329, - 0.0008118045, - 0.00896132, - -0.021625688, - 0.04505797, - -0.030714532, - 0.001798626, - 0.02301087, - -0.016400723, - 0.012461684, - 0.084329695, - 0.033947423, - 0.0033886805, - 0.024880258, - 0.040457603, - 0.011869054, - 0.036302757, - 0.019259617, - -0.04323633, - 0.01016828, - -0.018560894, - -0.02603231, - -0.037176747, - 0.036248654, - 0.061488714, - -0.034817405, - 0.027134847, - 0.019916616, - -0.026030567, - -0.17207955, - 0.030294493, - 0.025550704, - 0.019697066, - 0.0120773995, - -0.012691998, - -0.021480735, - -0.020040315, - -0.046375185, - 0.03796013, - -0.06322813, - -0.05399097, - -0.022879183, - -0.0067980997, - 0.04740681, - 0.022155577, - -0.004736322, - -0.0014450552, - -0.029361691, - -0.009039012, - -0.05187258, - 0.004528676, - 0.023304092, - 0.013138474, - -0.009009597, - 0.0021194455, - 0.020138947, - -0.024238044, - -0.00252644, - 0.01949502, - -0.005693967, - -0.0031609635, - 0.00047767212, - 0.018764313, - -0.017518612, - -0.0045243553, - -0.008981155, - -0.0072661415, - 0.008834193, - -0.029114552, - 0.02463106, - 0.07608546, - -0.004110012, - 0.028456254, - 0.02502078, - -0.026397288, - -0.014281513, - -0.030943176, - -0.045873113, - -0.0438763, - -0.036031656, - -0.019698508, - -0.0076750857, - 0.032183427, - -0.07022402, - 0.00677586, - 0.0009350075, - 0.032040097, - 0.034964126, - 0.023399055, - 0.00039395838, - -0.026182398, - 0.02339147, - -0.051522505, - 0.006419993, - -0.01467893, - 0.056414183, - -0.014274381, - 0.011415257, - -0.034448896, - 0.026943136, - -0.041795168, - 0.03407763, - 0.021582616, - -0.0030778935, - 0.0091854185, - -0.039134458, - -0.022431368, - 0.0033517058, - -0.11140594, - 0.016788866, - 0.011444995, - 0.024499895, - -0.00086195493, - -0.036194764, - -0.070865065, - 0.012270462, - -0.031136945, - 0.03481045, - 0.26673514, - 0.0048267795, - -0.020487886, - 0.057633284, - 0.010193855, - -0.020563407, - -0.014300299, - 0.07764238, - -0.014329802, - -0.013388632, - 0.020754205, - 0.034195956, - 0.027580399, - 0.0031598795, - 0.028510934, - 0.019713469, - -0.049448375, - 0.005708092, - 0.0490325, - -0.003206488, - 0.00844976, - -0.02254413, - 0.021620598, - 0.0050802585, - -0.038230564, - -0.036401823, - -0.004372875, - -0.010507128, - 0.0048447074, - 0.0650347, - -0.02155753, - 0.024728747, - 0.03266111, - -0.029489044, - -0.07477838, - 0.018160071, - 0.050032277, - 0.009285432, - -0.023625806, - 0.006457939, - -0.037101757, - -0.011428502, - -0.034958832, - 0.012344001, - -0.016206305, - -0.002524026, - -0.032018565, - -0.021426557, - -0.05146144, - 0.016052939, - 0.025400624, - -0.011394453, - 0.015693132, - -0.036642604, - 0.0068125273, - -0.031102354, - 0.0006415432, - -0.015234702, - 0.0005981847, - 0.024656652, - 0.013792873, - 0.012591731, - -0.047468007, - 0.005383405, - -0.026767764, - 0.0010417238, - 0.01260241, - 0.009144099, - 0.052271474, - 0.03758399, - 0.014113168, - 0.027975073, - 0.0031395108, - 0.019137679, - 0.012002146, - 0.015285317, - 0.0697657, - 0.025773382, - -0.039341886, - -0.007636981, - -0.013763273, - -0.04143567, - -0.008759471, - -0.008617591, - -0.004905359, - -0.0006550477, - -0.0060989074, - 0.06635398, - 0.01786153, - 0.023639463, - 0.0017929941, - 0.005695068, - -0.01750974, - 0.08869328, - -0.02346365, - 0.017725026, - 0.02551195, - -0.024749767, - -0.038333807, - -0.011524219, - -0.03871031, - -0.04890368, - -0.023725832, - 0.006170773, - 0.006687999, - 0.027459502, - -0.0144076375, - 0.026287941, - -0.020721149, - 0.0149823865, - -0.0404207, - 0.014675915, - 0.019121645, - -0.020943886, - -0.016600523, - 0.08309284, - 0.02306875, - 0.0207298, - 0.060951415, - 0.012785473, - -0.044505276, - 0.041076746, - 0.006343953, - 0.05000875, - -0.033053014, - -0.05189499, - -0.016196232, - 0.037317615, - -0.020528756, - 0.048521858, - 0.0059099253, - -0.028122889, - 0.0054981792, - 0.032670267, - 0.04803307, - -0.00087276264, - 0.03507837, - 0.005714917, - 0.0055141556, - 0.028890997, - -0.0016351967, - 0.006987805, - -0.018262777, - 0.029791636, - 0.0046638516, - 0.033249326, - -0.03445243, - -0.01295842, - 0.01441743, - 0.029409211, - 0.016255874, - 0.075089335, - 0.038607463, - -0.011865004, - -0.013541799, - -0.049965438, - -0.030712731, - -0.0015737375, - 0.00763361, - 0.014510599, - -0.010450412, - -0.0054039606, - 0.013869371, - 0.024631467, - -0.031981017, - 0.02500048, - 0.023703776, - 0.03888945, - -0.025448658, - 0.0066534434, - -0.009602537, - -0.050349317, - -0.034438644, - 0.03233763, - -0.02035359, - 0.03757376, - -0.034054853, - -0.045944143, - -0.011253913, - -0.025049351, - 0.018554306, - -0.0058857477, - -0.01725867, - -0.031350374, - 0.029627116, - -0.020733485, - 0.033612225, - -0.0001879455, - 0.001458068, - -0.038463153, - -0.013755852, - 0.12936483, - 0.025318006, - -0.013488891, - 0.0211201, - -0.008994673, - 0.054140277, - -0.014247034, - 0.010100081, - 0.004439559, - 0.019165436, - -0.01293228, - 0.013440812, - -0.01457884, - -0.022894854, - -0.0016506446, - 0.0061472687, - -0.0019743817, - -0.04432136, - -0.027698763, - 0.020786721, - 0.02901247, - -0.036669828, - 0.026799718, - 0.0049706534, - -0.032803133, - 0.026451163, - 0.061354123, - 0.03629118, - -0.035489112, - 0.03024852, - -0.021444995, - -0.026118552, - -0.04047567, - -0.013171445, - 0.0036920856, - 0.027079854, - -0.040054124, - 0.0052097808, - -0.02069143, - -0.010579754, - -0.026185164, - 0.035094798, - -0.05246998, - -0.0048390855, - -0.039560266, - 0.004901103, - 0.031820156, - -0.002581134, - -0.018233713, - 0.04174489, - -0.005909926, - 0.032968145, - 0.03805019, - 0.0241604, - 0.015945574, - -0.065295294, - 0.018397937, - 0.01724435, - 0.006362389, - -0.048789244, - -0.008711931, - 0.0026172544, - -0.07219876, - 0.0053889975, - 0.014360534, - -0.004241924, - -0.0032152962, - -0.05656201, - -0.03285138, - -0.03315872, - 0.06317837, - -0.01305612, - -0.017421898, - -0.027925925, - -0.05318094, - 0.01995136, - -0.044360217, - -0.023339093, - -0.029853659, - -0.0005349371, - 0.011237946, - 0.0023761003, - -9.777695e-05, - 0.017065378, - 0.008594363, - -0.019714532, - -0.0017668798, - 0.047101125, - -0.015230747, - -0.06977736, - -0.01593241, - 0.013606452, - -0.0019426926, - -0.007321555, - -0.006290232, - 0.04356458, - -0.029171923, - -0.018613918, - -0.031129446, - 0.05053018, - 0.014480598, - -0.044001196, - 0.018877143, - -0.01576236, - 0.001698718, - -0.015324685, - 0.077233694, - -0.00790072, - -0.0030271024, - 0.020865494, - 0.0017506607, - -0.01973603, - -0.008550953, - -0.035506316, - 0.025520213, - -0.03349575, - -0.031800956, - 0.029868, - 0.015730975, - -0.023582421, - -0.03308246, - -0.02042539, - 0.009700147, - 0.05318986, - -0.036836386, - 0.019110564, - -0.050924964, - 0.019646002, - 0.02172725, - 0.018214362, - 0.020434529, - -0.009510095, - 0.004531248, - -0.05996597, - 0.060739607, - -0.02196907, - -0.025661202, - -0.030605223, - -0.00063389336, - -0.040063895, - -0.001662116, - 0.0028458296, - -0.03809297, - 0.007532881, - 0.013903948, - 0.032967746, - 0.0011178405, - 0.02572339, - 0.010090046, - 0.025032775, - -0.034541212, - 0.0490119, - -0.007415879, - -0.021755945, - 0.009419693, - 0.012542628, - -0.033163227, - 0.0025159218, - -0.020397192, - 0.035166726, - 0.00480968, - -0.02023163, - -0.036315314, - -0.00095725077, - -0.03425355, - -0.022871329, - 0.004085877, - 0.016833203, - 0.028805165, - -0.0076796613, - 0.034964815, - -0.018549174, - -0.03409035, - 0.0042313896, - -0.025309384, - 0.00086933566, - -0.0067311106, - 0.015848443, - 0.009354085, - -0.0075304187, - -0.0050861966, - -0.010490597, - 0.006780659, - -0.030111965, - 0.03158741, - -0.04444453, - -0.016070886, - 0.032569155, - -0.019903107, - -0.03140162, - -0.060445823, - -0.017239368, - 0.02532584, - -0.016167594, - -0.028979613, - -0.026047565, - 0.007677287, - -0.010623334, - -0.005742986, - -0.024515554, - 0.017910937, - -0.027402997, - 0.03715235, - -0.16124795, - 0.004937735, - -0.0028726917, - 0.053121883, - -0.028584244, - 0.015552479, - 0.005151478, - -0.0028693315, - 7.4971125e-05, - -0.0172123, - 0.004214222, - -0.014687752, - 0.014326957, - -0.01886226, - -0.004525692, - 0.030321203, - -0.012055664, - -0.00053526106, - -0.012307759, - 0.03298414, - -0.011449256, - -0.008144446, - 0.03446861, - -0.026350394, - 0.007076595, - 0.036833167, - -0.026910506, - 0.012592906, - -0.013327422, - 0.0037525764, - 0.029099714, - -0.029344535, - 0.023529947, - 0.020716734, - 0.024934646, - 0.053604923, - 0.031444784, - 0.020586275, - 0.038974516, - -0.0060891346, - -0.015166958, - -0.010662363, - -0.020694856, - -0.035508987, - -0.021125458, - 0.025793135, - 0.001245229, - -0.007885029, - -0.03348208, - 0.01604755, - -0.012663853, - 0.016712101, - -0.07827245, - 0.02121564, - -0.02697736, - 0.0024977273, - -0.010425861, - 0.017344903, - -0.02350563, - 0.023465443, - -0.048638504, - 0.027437428, - -0.016334001, - -0.010520566, - -0.03173934, - 0.039972708, - -0.10125317, - 0.0059645423, - 0.0021333932, - 0.006329766, - -0.040584467, - -0.024449449, - -0.023267766, - 0.0005388384, - 0.023606084, - 0.025171466, - 0.033454485, - -0.004060982, - -0.030102212, - -0.004214083, - 0.009479132, - -0.023248918, - -0.01711765, - 0.041533716, - 0.053464, - -0.018355522, - 0.0010048773, - 0.020233225, - -0.03397556, - -0.025564512, - -0.019184228, - -0.049973786, - 0.017904658, - 0.030558312, - -0.0060713724, - -0.0011789178, - -0.03647051, - 0.004114137, - -0.01747355, - -0.0024815144, - 0.014497628, - -0.023431826, - 0.025701253, - 0.00094681577, - -0.035608713, - 0.027759047, - -0.0048973016, - 0.0028433448, - 0.01714142, - 0.0040114406, - -0.08186222, - 0.001504233, - -0.038464736, - 0.006342804, - -0.055073783, - -0.013568032, - 0.043404534, - 0.005846853, - 0.0010117785, - 0.037113357, - -0.031810734, - -0.018276729, - -0.036663122, - -0.028997479, - 0.025806835, - 0.027248897, - 0.027375758, - 0.044399276, - 0.029030504, - -0.04426898, - 0.017303139, - -0.05697623, - -0.0048485114, - -0.013266216, - 0.032227486, - -0.0015718974, - -0.0041465308, - 0.062229417, - -0.0476542, - -0.0393459, - -0.06611899, - -0.009516534, - -0.014097821, - -0.06326271, - 0.022630574, - 0.0105031645, - 0.017197985, - -0.0029821533, - 0.018753763, - -0.04362471, - -0.02358172, - 0.023925336, - -0.020253649, - 0.035189502, - 0.028137224, - 0.02796643, - -0.05082343, - 0.030518577, - -0.026187275, - 0.049044814, - -0.0014382052, - -0.0071974313, - -0.019514164, - -0.04678933, - 0.0038900028, - 0.04175545, - -0.020044949, - 0.004461038, - -0.041491985, - 0.01472482, - 0.041775912, - 0.04120755, - -0.011634551, - -0.019989306, - 0.04412291, - -0.004687343, - 0.026644224, - 0.006771995, - 0.02421993, - -0.009521988, - 0.03603204, - -0.034742337, - 0.028528385, - 0.02376243, - 0.022733629, - 0.03731325, - 0.005610845, - 0.025229195, - 0.033122566, - -0.011691628, - -0.0012906544, - -0.022317326, - 0.026396208, - -0.019081533, - 0.020932008, - 0.023135103, - 0.008220864, - -0.022540981, - -0.0045254338, - 0.024875924, - -0.027374046, - 0.043750864, - -0.004198584, - -0.003487473, - 0.02688751, - -0.021254478, - -0.0051326077, - -0.028950123, - -0.024273885, - 0.0038261758, - 0.028979441, - 0.00300767, - -0.022982204, - 0.014468483, - -0.030607553, - -0.039471906, - 0.0543203, - -0.029934777, - 0.013926918, - 0.0038656201, - 0.033895873, - 0.027200352, - 0.037890792, - -0.041717835, - -0.056469403, - -0.0342521, - 0.011269026, - -0.0069762636, - -0.015309743, - 0.0070049106, - -0.05005616, - -0.020231502, - 0.009734973, - -0.009714966, - 0.038587984, - 0.0020214266, - -0.04298902, - 0.017955454, - -0.015262794, - -0.0022945707, - 0.06333846, - 0.045666393, - 0.029671278, - -0.0288319 + -0.03485036, + -0.006523965, + -0.024870101, + -0.021951621, + -0.04676112, + 0.0019194462, + 0.024405016, + 0.035693713, + -0.008592627, + 0.026169837, + -0.012792799, + -0.01111802, + -0.020802893, + -0.013956348, + -0.0027344371, + 1.18717335e-05, + 0.016262613, + 0.02039375, + -0.010071774, + -0.01875823, + -0.018803325, + -0.009060597, + 0.013147215, + 0.028142855, + -0.04177979, + 0.02827999, + -0.012310784, + -0.06533352, + -0.025828896, + 0.04129906, + -0.003782613, + -0.022862246, + 0.033427875, + -0.0054732924, + -0.038241345, + -0.029082129, + -0.013977575, + -0.01715362, + -0.08533639, + 0.00830145, + -0.03488972, + 0.010252315, + -0.005535598, + -0.017507868, + 0.0075469627, + -0.03480188, + -0.000507779, + -0.023278741, + -0.020672912, + -0.06980631, + -0.023200769, + -0.004430873, + 0.011973691, + -0.017140994, + 0.044351693, + 0.043278854, + -0.042956855, + -0.014997362, + -0.043559704, + 0.023506247, + -0.053146906, + -0.010448102, + -0.020760927, + 0.013700241, + 0.0028324132, + 0.030927282, + 0.040706042, + -0.00079867116, + -0.0016535603, + -0.029102279, + -0.0036018426, + 0.0068588485, + -0.0143683525, + -0.017418599, + -0.06759086, + 0.04024998, + 0.0013544558, + -0.027406689, + -0.0076633324, + 0.028525572, + -0.040828053, + -0.006313824, + 0.055598367, + 0.0038820985, + -0.0057513574, + 0.029751923, + -0.02152409, + 0.045518126, + 0.056295734, + -0.014960843, + -0.023993399, + -0.03368726, + 0.05779403, + -0.046619095, + -0.03961416, + 0.004086139, + 0.002741565, + -0.028147021, + 0.06516865, + 0.016405731, + -0.010616796, + 0.018252261, + 0.005795998, + -0.0053562457, + 0.04854147, + 0.04298808, + 0.019046724, + -0.0052411426, + -0.004604102, + -0.011814224, + -0.009750135, + 0.015850324, + 0.03265538, + 0.020712115, + -0.006471794, + -0.04911774, + 0.01483612, + -0.01579769, + -0.013883977, + -0.015874514, + 0.03148097, + 0.04799249, + 0.053061053, + -0.037908312, + 0.04927149, + 0.0094831465, + 0.027591636, + 0.03283308, + -0.0038896052, + 0.033055276, + 0.027255742, + 0.03658406, + -0.015679047, + -0.0034444723, + -0.08805762, + -0.006555244, + -0.0034031516, + 0.035184074, + 0.04333371, + -0.0530124, + 0.003475752, + 0.023959255, + -0.0341101, + -0.026977474, + 0.0306026, + -0.04823844, + 0.005313242, + 0.04337344, + -0.016344374, + -0.052851405, + 0.025235528, + -0.008192982, + 0.042521715, + 0.03924706, + -0.017751662, + -0.06547015, + 0.015605447, + 0.0025222667, + 0.052172974, + -0.01636314, + 0.005909687, + 0.0063013383, + -0.031018585, + 0.024667436, + 0.04233253, + -0.013303014, + 0.013495011, + -0.011126833, + -0.038032837, + -0.0042188154, + 0.008843474, + -0.037746347, + 0.008016975, + 0.013132684, + 0.011910548, + 0.011193239, + 0.060552154, + 0.01644624, + -0.011260529, + -0.028127898, + -0.033737317, + -0.009595608, + -0.012269939, + -0.017357165, + -0.021629686, + 0.017643092, + -0.029173048, + 0.0032606227, + -0.008527376, + 0.05322887, + 0.013322519, + -0.010476832, + 0.03258906, + 0.03202706, + -0.03405511, + -0.018965025, + 0.026686324, + -0.0035002579, + 0.0034975614, + -0.027884498, + 0.022383856, + 0.034188, + 0.045019623, + -0.016051197, + -0.045693796, + -0.018393034, + -0.015090342, + -0.038736198, + 0.007350469, + -0.013768265, + 0.022838596, + 0.0048558167, + -0.007734886, + -0.022188174, + -0.023705874, + -0.016501134, + 0.0037238947, + -0.012784965, + 0.011435852, + -0.026673306, + -0.005509241, + 0.02364078, + 0.0025442347, + -0.009661722, + 0.0073032924, + 0.016039537, + 0.031440396, + 0.004281652, + -0.03770763, + 0.028144414, + -0.0010302472, + 0.031301018, + 0.03395194, + -0.0014075655, + 0.01414747, + -0.049955558, + -0.0012861338, + 0.017909067, + 0.028062966, + -0.02871389, + 0.04881674, + -0.063322306, + 0.038415723, + -0.011686032, + -0.042711668, + -0.017449714, + 0.002141979, + 0.0457172, + -0.04424886, + -0.017762464, + -0.006207608, + 0.01629242, + -0.037824757, + -0.011094415, + 0.014849885, + 0.0038221972, + 0.008234737, + 0.026768895, + -0.0046303924, + 0.01726791, + -0.014142183, + 0.019061428, + 0.01821286, + 0.024755321, + 0.027596371, + -0.011208457, + 0.0010563019, + -0.023535619, + -0.010621036, + -0.00074850395, + -0.0036844595, + -0.019430794, + 0.04973371, + -0.020635622, + -0.0047764294, + 0.019615741, + -0.018160922, + 0.017911669, + 0.08115741, + 0.026115179, + -0.006572227, + 0.012608993, + 0.045932107, + 0.0027327065, + 0.035970118, + 0.008203159, + -0.030672822, + 0.014164894, + -0.031151213, + -0.03182053, + -0.028612746, + 0.021081073, + 0.066009685, + -0.04395738, + 0.022311244, + 0.011704254, + -0.03953015, + -0.17731258, + 0.02446469, + 0.02200765, + 0.033863176, + 0.009451169, + -0.009153969, + -0.024325192, + -0.016987937, + -0.03815129, + 0.03530359, + -0.07062162, + -0.056043025, + -0.0111074345, + -0.00892553, + 0.04874841, + 0.022275368, + -0.00049478165, + -0.010223968, + -0.014345399, + -0.012954244, + -0.054299448, + -0.014125319, + 0.023162726, + 0.021651436, + -0.012387239, + 0.0051369322, + 0.031682234, + -0.019705169, + -0.0074669328, + 0.02000536, + -0.008161489, + -0.0100305565, + -0.0013780366, + 0.028171338, + -0.009415919, + 0.0021736685, + 0.010446073, + -0.010456459, + 0.009267223, + -0.02139086, + 0.043684043, + 0.07192079, + -0.007597984, + 0.02457813, + 0.015363794, + -0.025522145, + 0.0014028245, + -0.023182474, + -0.053059332, + -0.02515991, + -0.033534322, + -0.024742566, + 0.009469918, + 0.02868977, + -0.06879678, + 0.004279142, + -0.021392403, + 0.029523512, + 0.0140930265, + 0.040809836, + -0.0034976192, + -0.027779879, + 0.024514323, + -0.07213847, + -0.0006524821, + -0.02826045, + 0.048646033, + -0.00341837, + 0.011114654, + -0.031905606, + 0.019060064, + -0.05665736, + 0.02308151, + 0.017008655, + -0.002823673, + 0.011452782, + -0.03378409, + -0.009196477, + 0.0034898797, + -0.111773275, + 0.006144034, + 0.019115357, + 0.023291895, + -0.0086863665, + -0.03703144, + -0.062035933, + 0.0115044, + -0.021629985, + 0.01940764, + 0.2651562, + -0.0028407408, + -0.031331632, + 0.04184685, + 0.0192927, + -0.018956458, + -0.016081788, + 0.079190314, + -0.012182804, + -0.012182674, + 0.030636767, + 0.03426719, + 0.016827187, + -0.0012515389, + 0.02072367, + 0.02729455, + -0.044176634, + 0.001213331, + 0.06003864, + -0.010951624, + 0.011007386, + -0.0120707825, + 0.01967973, + 0.0095139835, + -0.0331325, + -0.03138036, + -0.005941679, + -0.017965306, + 0.0039845016, + 0.055597417, + -0.028543742, + 0.025939127, + 0.04222716, + -0.027559107, + -0.073364444, + 0.028254727, + 0.047389302, + 0.011136994, + -0.02329983, + -0.0014732265, + -0.032746255, + -0.011333437, + -0.050461497, + 0.009710012, + -0.0007233123, + -0.0023836573, + -0.032397136, + -0.029447773, + -0.03913593, + 0.022129782, + 0.022849508, + -0.0028128333, + 0.022123925, + -0.044084348, + 0.003719973, + -0.042123318, + -0.0056991354, + -0.00852776, + 0.00312135, + 0.023790535, + 0.011400686, + 0.011409521, + -0.05798747, + 0.011576273, + -0.010227346, + 0.005631751, + 0.007495863, + 0.00804295, + 0.076911, + 0.023302251, + 0.011465635, + 0.017791932, + -0.0011115364, + 0.01702136, + 0.0064648306, + 0.017583003, + 0.064273424, + 0.03191352, + -0.04347176, + -0.0084685655, + -0.007889204, + -0.04106867, + -0.015657114, + -0.022527304, + 0.0010747634, + -0.009583432, + -0.01001918, + 0.057714473, + 0.012642176, + 0.014514113, + 0.008418106, + 0.0029056633, + -0.02109183, + 0.08549639, + -0.02401644, + 0.008622991, + 0.009177995, + -0.027816093, + -0.03818879, + -0.012544949, + -0.048338052, + -0.059360508, + -0.026988946, + 0.012060502, + 0.00735656, + 0.01653607, + -0.0148168625, + 0.028525626, + -0.020907076, + 0.028334254, + -0.033567596, + 0.019952297, + 0.01571829, + -0.030224144, + -0.0078595085, + 0.07016056, + 0.030943854, + 0.018744482, + 0.06552349, + 0.019052256, + -0.048748452, + 0.040406156, + 0.008260294, + 0.038071383, + -0.032423597, + -0.042639453, + -0.022628836, + 0.036644712, + -0.011333548, + 0.05039947, + 0.011461258, + -0.02223915, + 0.009473547, + 0.026953608, + 0.036760084, + 0.0017486485, + 0.013157991, + 0.0059267255, + 0.007039213, + 0.03609918, + -0.018248925, + 0.018680604, + -0.02767531, + 0.03577312, + 0.00055750203, + 0.02181149, + -0.028532414, + -0.011596229, + -0.0005457933, + 0.035079267, + 0.02135478, + 0.07462979, + 0.034515195, + 0.004064192, + 0.001072951, + -0.04284846, + -0.030484488, + -0.0036550527, + 0.006420163, + 0.012750347, + -0.012521361, + -0.01923254, + 0.015273171, + 0.02544576, + -0.015367402, + 0.019597257, + 0.02435614, + 0.032175273, + -0.038116544, + -0.00093479076, + 0.0013199619, + -0.048687097, + -0.028815554, + 0.032724977, + -0.029785093, + 0.016622232, + -0.018459229, + -0.053232837, + -0.013562255, + -0.030861689, + 0.026351627, + -0.0030361135, + -0.00872005, + -0.033535004, + 0.03025441, + -0.014477477, + 0.050545838, + -0.0028624274, + 0.004125899, + -0.032477897, + -0.008121306, + 0.12731615, + 0.026881339, + -0.014786382, + 0.035495725, + 0.00082117005, + 0.04081602, + -0.013261213, + 0.0069885044, + 0.013010508, + 0.006064077, + -0.030903772, + 0.015621737, + -0.025409952, + -0.019215796, + 0.008197931, + -0.0059183026, + -0.0133599, + -0.058643367, + -0.016191332, + 0.021001328, + 0.022470305, + -0.01905571, + 0.034924906, + 0.0012492832, + -0.027229425, + 0.028659761, + 0.050299875, + 0.028339231, + -0.032533582, + 0.03901179, + -0.015456914, + -0.012930993, + -0.029640103, + -0.0027097326, + 0.0042801523, + 0.031620346, + -0.049821004, + 0.013133078, + -0.01899334, + -0.007282641, + -0.035775866, + 0.016878197, + -0.053011406, + -0.007267962, + -0.03583142, + 0.018804934, + 0.036747944, + -0.009275369, + -0.021193573, + 0.048727028, + -0.006180164, + 0.029412761, + 0.02580566, + 0.025216166, + 0.011644537, + -0.064970955, + 0.022830062, + 0.0027662478, + 0.024282526, + -0.04159977, + -0.003701238, + 0.006597354, + -0.06000885, + 0.008319213, + 0.018260457, + -0.009221352, + -0.006496549, + -0.06236123, + -0.047565293, + -0.029486923, + 0.06445119, + -0.013600725, + -0.01965479, + -0.02777698, + -0.052896906, + 0.018137082, + -0.04071823, + -0.0169595, + -0.025429947, + 0.0036767197, + 0.011745697, + 0.0048488337, + -0.00027546255, + 0.015787488, + 0.007459889, + -0.030069385, + 0.0021149854, + 0.05602186, + -0.015326911, + -0.06182895, + -0.0011165651, + 0.012864129, + -0.0016939089, + 0.0067999484, + 0.0010948898, + 0.033293694, + -0.031025292, + -0.017793514, + -0.027720565, + 0.04350979, + 0.01690967, + -0.039987262, + 0.01068626, + -0.022370197, + -0.004096849, + -0.012856249, + 0.06924249, + -0.0065072216, + -0.000597387, + 0.019755809, + 0.008910962, + -0.00568093, + -0.008516902, + -0.033760957, + 0.016812917, + -0.027316146, + -0.026889825, + 0.015981369, + 0.017211344, + -0.01262618, + -0.02413841, + -0.008612607, + 0.0055217566, + 0.055153783, + -0.035130538, + 0.020660555, + -0.049755167, + 0.024131725, + 0.020430876, + 0.023211103, + 0.03711559, + -0.019059062, + 0.0022703004, + -0.0505399, + 0.053637497, + -0.024510836, + -0.014918175, + -0.027184827, + 0.0034064827, + -0.049552877, + -0.0017316656, + 0.0044178544, + -0.040789127, + 0.0199112, + 0.015300785, + 0.017422635, + 0.010424917, + 0.03423134, + 0.00052812154, + 0.021104172, + -0.051543217, + 0.037585888, + -0.001752495, + -0.019554261, + 0.013937136, + 0.013765535, + -0.03148354, + -0.0056180186, + -0.017817277, + 0.03326038, + -0.00013004264, + -0.02274853, + -0.03417097, + -0.0066750236, + -0.0075843916, + -0.019918067, + 0.01152048, + 0.021681871, + 0.018207202, + -0.0020919684, + 0.030538194, + -0.012318519, + -0.031633995, + 0.00013115391, + -0.03362657, + 0.00907644, + 0.007918352, + 0.010078273, + 0.014963157, + -0.012318672, + -0.007231364, + -0.01027362, + 0.005270827, + -0.03693281, + 0.024665194, + -0.041493226, + -0.017185101, + 0.02431877, + -0.032249715, + -0.026663367, + -0.04970651, + -0.02164517, + 0.027656583, + -0.022452675, + -0.02569411, + -0.03141506, + 0.00958662, + -0.010624525, + 0.0013384996, + -0.017067885, + 0.010905406, + -0.02574669, + 0.027742615, + -0.16543303, + 0.0044359285, + -0.0041801147, + 0.050032545, + -0.036420442, + 0.016444901, + 0.004906219, + 0.00305755, + 0.000645408, + -0.02585195, + 5.851421e-05, + -0.0177179, + 0.017901, + -0.014389969, + -0.014035693, + 0.031114766, + -0.009225222, + 0.003178253, + -0.011938513, + 0.029872155, + -0.01868456, + -0.0047811917, + 0.04640891, + -0.028970834, + -0.003929274, + 0.037099987, + -0.03288008, + 0.025929831, + -0.017864862, + -0.001104512, + 0.029252192, + -0.03019128, + 0.021096261, + 0.039015904, + 0.020269977, + 0.055913292, + 0.029641623, + 0.016977029, + 0.037405808, + -0.0060188845, + -0.012283051, + -0.009624288, + -0.024535663, + -0.03902192, + -0.008151951, + 0.027306039, + 0.003717378, + -0.0055695516, + -0.030805841, + 0.021994106, + -0.017613834, + 0.02414003, + -0.07084169, + 0.02342745, + -0.025662372, + 0.00950205, + -0.0128408745, + 0.028426634, + -0.024287108, + 0.018353976, + -0.05276756, + 0.032767028, + -0.028602868, + -0.010929256, + -0.025840053, + 0.03306693, + -0.10701591, + 0.0071515124, + -0.004364812, + 0.017506683, + -0.026777966, + -0.025047412, + -0.012522828, + -0.003158993, + 0.01915694, + 0.0080304975, + 0.046226174, + 0.0031628187, + -0.028501768, + 0.0017846226, + 0.011096759, + -0.038458068, + -0.028518906, + 0.02648644, + 0.05748379, + -0.0052262894, + 0.00090398826, + 0.019646378, + -0.034181133, + -0.031019276, + -0.021406053, + -0.038842995, + 0.029685358, + 0.026918044, + -0.008026107, + 0.017391609, + -0.039732188, + 0.01055362, + -0.010321314, + -0.0028208157, + 0.019798337, + -0.030184537, + 0.037872516, + 0.0022543075, + -0.024099082, + 0.024197003, + -0.012363028, + -0.0036628547, + 0.0106351515, + 0.004500302, + -0.074983455, + -0.0024789374, + -0.039768178, + 0.007209211, + -0.07402429, + -0.0035831241, + 0.052372623, + 0.008074728, + -0.010254601, + 0.03723895, + -0.0337514, + -0.024283621, + -0.027402911, + -0.03691343, + 0.021020405, + 0.020925667, + 0.029383194, + 0.04252137, + 0.028472798, + -0.04295308, + 0.021773832, + -0.045806758, + -0.0066437596, + -0.0014096142, + 0.042101096, + 0.00016596373, + -0.00016136566, + 0.060364816, + -0.044781003, + -0.042765927, + -0.0654631, + -0.0038248457, + -0.010606115, + -0.05439583, + 0.024176728, + 0.022730034, + 0.03621591, + 0.0009763813, + 0.034391966, + -0.04052909, + -0.032464515, + 0.020484608, + -0.027862048, + 0.030571727, + 0.02789852, + 0.026415614, + -0.057382565, + 0.02988672, + -0.03243118, + 0.046732627, + 0.013446517, + -0.019803952, + -0.022840079, + -0.038886994, + -0.011342753, + 0.042946495, + -0.022382729, + 0.0076412205, + -0.045635633, + 0.01405105, + 0.025229704, + 0.06655055, + -0.010767731, + -0.0059707374, + 0.03609742, + -0.005530074, + 0.03376942, + 0.008308468, + 0.021039316, + -0.010723757, + 0.0355036, + -0.035310235, + 0.028243257, + 0.025039956, + 0.024774326, + 0.034099862, + 0.0011991276, + 0.017638503, + 0.026746906, + -0.019286534, + -0.0075161713, + -0.02140353, + 0.017149443, + -0.012274824, + 0.01085057, + 0.018384213, + 0.0033054748, + -0.023285326, + -0.015042774, + 0.028020097, + -0.02429766, + 0.038505014, + -0.010975393, + -0.0045849024, + 0.01914343, + -0.031177802, + 0.006690604, + -0.02940985, + -0.024704568, + -0.003650038, + 0.021816175, + 0.009830584, + -0.03013565, + 0.024328627, + -0.035891555, + -0.048526727, + 0.04881518, + -0.030377112, + 0.01964267, + 0.007051219, + 0.03593706, + 0.024328787, + 0.042422246, + -0.030494707, + -0.055859216, + -0.031119449, + 0.016314944, + -0.005308099, + -0.0041531087, + 0.0147686, + -0.04632526, + -0.01062353, + 0.0033541117, + -0.009182904, + 0.041490734, + 0.0035344528, + -0.029527742, + 0.02343805, + -0.0061483, + 2.2035414e-05, + 0.06465615, + 0.045795295, + 0.031973656, + -0.03712669 ] \ No newline at end of file diff --git a/embedding/pgvector_store.py b/embedding/pgvector_store.py new file mode 100644 index 00000000..ce10a66e --- /dev/null +++ b/embedding/pgvector_store.py @@ -0,0 +1,123 @@ +import json +import os +from contextlib import contextmanager +from typing import Any + +import psycopg2 +from dotenv import load_dotenv +from psycopg2.extras import RealDictCursor + +load_dotenv() + +DEFAULT_TABLE = "public.es3_vector" + + +def _embedding_dim() -> int: + return int(os.getenv("EMBEDDING_DIM", "1024")) + + +def _table_name() -> str: + return os.getenv("PGVECTOR_TABLE", DEFAULT_TABLE) + + +def _connection_kwargs() -> dict[str, Any]: + return { + "host": os.getenv("PGVECTOR_DB_HOST", os.getenv("DB_HOST", "localhost")), + "port": os.getenv("PGVECTOR_DB_PORT", os.getenv("DB_PORT", "5432")), + "user": os.getenv("PGVECTOR_DB_USER", os.getenv("DB_USER", "example_db_user")), + "password": os.getenv("PGVECTOR_DB_PASSWORD", os.getenv("DB_PASSWORD", "example_db_password")), + "dbname": os.getenv("PGVECTOR_DB_NAME", os.getenv("DB_NAME", "example_db_name")), + } + + +def _vector_literal(vector: list[float]) -> str: + if not isinstance(vector, list) or not vector: + raise ValueError("embedding vector is empty") + return "[" + ",".join(str(float(value)) for value in vector) + "]" + + +@contextmanager +def get_connection(): + conn = psycopg2.connect(**_connection_kwargs()) + try: + yield conn + finally: + conn.close() + + +def ensure_schema(dim: int | None = None) -> None: + dim = dim or _embedding_dim() + table = _table_name() + try: + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute("CREATE EXTENSION IF NOT EXISTS vector") + cur.execute( + f""" + CREATE TABLE IF NOT EXISTS {table} ( + id BIGSERIAL PRIMARY KEY, + document TEXT NOT NULL, + embedding vector({dim}) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + except psycopg2.Error as exc: + message = str(exc).strip() + if "extension" in message and "vector" in message: + cfg = _connection_kwargs() + raise RuntimeError( + "pgvector extension is not installed on the connected PostgreSQL server " + f"({cfg['host']}:{cfg['port']}/{cfg['dbname']}). " + "Use the pgvector/pgvector:pg16 container from ExtS3-Demo/docker-compose.yml " + "or install pgvector on that PostgreSQL instance." + ) from exc + raise + + +def count_vectors() -> int: + ensure_schema() + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute(f"SELECT COUNT(*) FROM {_table_name()}") + return int(cur.fetchone()[0]) + + +def insert_vector_record(document: str | dict[str, Any], embedding: list[float]) -> None: + ensure_schema(len(embedding)) + document_text = json.dumps(document, ensure_ascii=False, sort_keys=True) if isinstance(document, dict) else document + vector_text = _vector_literal(embedding) + with get_connection() as conn: + with conn.cursor() as cur: + cur.execute( + f"INSERT INTO {_table_name()} (document, embedding) VALUES (%s, %s::vector)", + (document_text, vector_text), + ) + conn.commit() + + +def search_vectors( + embedding: list[float], + *, + match_threshold: float = 0.0, + match_count: int = 10, +) -> list[dict[str, Any]]: + ensure_schema(len(embedding)) + vector_text = _vector_literal(embedding) + with get_connection() as conn: + with conn.cursor(cursor_factory=RealDictCursor) as cur: + cur.execute( + f""" + SELECT + id::text AS id, + document, + 1 - (embedding <=> %s::vector) AS similarity + FROM {_table_name()} + WHERE 1 - (embedding <=> %s::vector) > %s + ORDER BY embedding <=> %s::vector + LIMIT %s + """, + (vector_text, vector_text, match_threshold, vector_text, match_count), + ) + return [dict(row) for row in cur.fetchall()] diff --git a/embedding/scenario/playwright_dynamic_harness.py b/embedding/scenario/playwright_dynamic_harness.py index 15928394..b006c05f 100644 --- a/embedding/scenario/playwright_dynamic_harness.py +++ b/embedding/scenario/playwright_dynamic_harness.py @@ -3,6 +3,7 @@ import hashlib import json import os +import platform import shutil import tempfile import threading @@ -406,7 +407,7 @@ def __init__( "wait_for_load_state_completed": False, "wait_for_load_state_error": "", "mock_server_check_attempted": False, - "mock_server_autostart_enabled": parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=False), + "mock_server_autostart_enabled": parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=True), "mock_server_autostarted": False, "mock_server_host": "", "mock_server_port": 0, @@ -1357,7 +1358,12 @@ def _ensure_context(self): self._execution["headless"] = bool(effective_headless) self._execution["display_env"] = str(os.environ.get("DISPLAY") or "") self._execution["xvfb_available"] = bool(shutil.which("xvfb-run") is not None) - self._execution["headed_supported"] = bool(self._execution["display_env"]) or bool(self._execution["xvfb_available"]) + desktop_os = platform.system().lower() in {"windows", "darwin"} + self._execution["headed_supported"] = ( + desktop_os + or bool(self._execution["display_env"]) + or bool(self._execution["xvfb_available"]) + ) if not effective_headless and not self._execution["headed_supported"]: self._execution["extension_load_error"] = "headed_mode_requires_display_or_xvfb" self._notes.append("Set DYNAMIC_HARNESS_HEADLESS=false and run the service under xvfb-run to test MV3 extensions in headed mode on a headless server.") @@ -1448,7 +1454,7 @@ def on_request(req): self._context.on("request", on_request) self._context.on("requestfinished", self._mark_request_finished) self._context.on("requestfailed", self._mark_request_failed) - if parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=False): + if parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=True): self._autostart_mock_server() def _current_observation(self) -> dict: @@ -1551,6 +1557,11 @@ def open_mock_page(self, action: dict) -> dict: self._execution["expected_target_url"] = str(url or "") self._execution["page_load_started"] = True + if parse_bool_env(os.getenv("DYNAMIC_MOCK_AUTOSTART"), default=True) and not self._execution.get("mock_server_autostarted"): + self._autostart_mock_server() + if self._execution.get("mock_server_url"): + url = str(self._execution["mock_server_url"]) + self._execution["expected_target_url"] = url emulated_host_ok = bool(self._emulated_target_host) and isinstance(url, str) and ( urlparse(str(url)).hostname or "" ).lower() == self._emulated_target_host diff --git a/main.py b/main.py index 422f3011..fee70bcc 100644 --- a/main.py +++ b/main.py @@ -75,6 +75,13 @@ def run_retro_monitor(): @asynccontextmanager async def lifespan(app: FastAPI): # 1. hm_new 홀딩 스케줄러 시작 + try: + from embedding.base_db import ensure_knowledge_base_seeded + + await run_in_threadpool(ensure_knowledge_base_seeded) + except Exception as seed_e: + print(f"[pgvector] startup seed check failed: {seed_e}") + hm_start() print("📢 [System] hm_new scheduler started.") @@ -154,7 +161,7 @@ def _browser_to_program_type(browser: str) -> str: def _decision_to_nexus_bucket(decision: str) -> str: lowered = str(decision or "").strip().lower() - if lowered in {"approve", "safe"}: + if lowered == "safe": return "safe" if lowered in {"reject", "high", "critical"}: return "review"