diff --git a/.gitignore b/.gitignore index 933ab6b4..80400157 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ analysis_result/ backend/results/ results_json/ storage/ +profiles/ # ExtAnalysis 분석 임시 디렉토리 및 결과 ExtAnalysis/lab/ diff --git a/ExtAnalysis/reports.json b/ExtAnalysis/reports.json index 8de08ae1..91319ba5 100644 --- a/ExtAnalysis/reports.json +++ b/ExtAnalysis/reports.json @@ -1231,6 +1231,41 @@ "report_directory": "\\EXA2026161145217", "time": "2026-06-10 14:52:18", "version": "5.7" + }, + { + "id": "EXA2026163053429", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163053429", + "time": "2026-06-12 05:34:30", + "version": "1.0.1" + }, + { + "id": "EXA2026163054842", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163054842", + "time": "2026-06-12 05:48:43", + "version": "1.0.1" + }, + { + "id": "EXA2026163062533", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163062533", + "time": "2026-06-12 06:25:35", + "version": "1.0.1" + }, + { + "id": "EXA2026163071425", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163071425", + "time": "2026-06-12 07:14:29", + "version": "1.0.1" + }, + { + "id": "EXA2026163071822", + "name": "Chrome MCP Server - AI Browser Control", + "report_directory": "/EXA2026163071822", + "time": "2026-06-12 07:18:23", + "version": "1.0.1" } ] } \ No newline at end of file diff --git a/backend/profile/__init__.py b/backend/profile/__init__.py new file mode 100644 index 00000000..644c0db6 --- /dev/null +++ b/backend/profile/__init__.py @@ -0,0 +1,26 @@ +"""Extension Profile: objective, version-by-version change history of an extension. + +Records what an extension *is* and how it changes between versions (manifest +facts, file hashes/sizes, diffs) — not analysis output. See ``builder`` for the +JSON generator. +""" + +from .builder import ( + build_profile, + build_snapshot, + compute_diff, + content_hash, + is_minified, + make_unified_diff, + validate_profile, +) + +__all__ = [ + "build_profile", + "build_snapshot", + "compute_diff", + "content_hash", + "is_minified", + "make_unified_diff", + "validate_profile", +] diff --git a/backend/profile/builder.py b/backend/profile/builder.py new file mode 100644 index 00000000..5537a954 --- /dev/null +++ b/backend/profile/builder.py @@ -0,0 +1,518 @@ +"""Extension Profile JSON generator. + +An Extension Profile records the *objective* state of a browser extension per +version and the diff against the previous version. It is deliberately NOT an +analysis-result store: scanner findings, embeddings, fingerprints, obfuscation +scores, capability inference and risk rationale are excluded. The only thing +borrowed from analysis is a thin ``verdict`` breadcrumb (risk_grade + result_id) +so a reader knows where to find the full result. + +Public API (the JSON-generation core): + build_snapshot - extension archive/dir -> objective snapshot (+ file bytes) + content_hash - stable hash over the file (path, sha256) set + is_minified - heuristic: is this file undiffable / minified? + make_unified_diff- unified diff text (+ truncation flag) for two texts + compute_diff - snapshot vs snapshot -> permission/manifest/file diff + build_profile - assemble/extend a profile document from a snapshot + validate_profile - jsonschema validation against extension-profile.schema.json + +Blob storage (Nexus) and DB persistence are out of scope here. Fetching previous +file bytes for inline diffs is delegated to a pluggable ``blob_loader`` callable; +without it, modified files fall back to pointer-only (blob_ref, diff=null). +""" + +from __future__ import annotations + +import hashlib +import json +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +SCHEMA_VERSION = "1.0" + +# Inline-diff guard rails: a single modified file should not blow up the profile. +MAX_DIFF_LINES = 2000 +MAX_DIFF_BYTES = 200_000 + +# Minified / undiffable heuristics. +_MINIFIED_LONGEST_LINE = 5000 +_MINIFIED_AVG_LINE = 500 +_MINIFIED_DENSE_AVG = 1000 +_MINIFIED_DENSE_MAX_LINES = 5 + +BlobLoader = Callable[[str], Optional[bytes]] +Snapshot = Dict[str, Any] +Profile = Dict[str, Any] + +_SCHEMA_CACHE: Optional[Dict[str, Any]] = None + + +# --------------------------------------------------------------------------- # +# hashing / small helpers +# --------------------------------------------------------------------------- # +def _sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _now_iso() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _sorted_unique(values: Any) -> List[str]: + if not isinstance(values, list): + return [] + return sorted({str(v) for v in values}) + + +def _norm_path(path: str) -> str: + return path.replace("\\", "/").lstrip("/") + + +# --------------------------------------------------------------------------- # +# content hash +# --------------------------------------------------------------------------- # +def content_hash(files: List[Dict[str, Any]]) -> str: + """Stable hash of the file set, keyed by ``':'`` entries. + + Order-independent: entries are sorted before hashing, so two extractions of + the same files always produce the same content_hash. + """ + entries = sorted(f"{f['path']}:{f['sha256']}" for f in files) + joined = "\n".join(entries) + return "sha256:" + hashlib.sha256(joined.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- # +# minified detection +# --------------------------------------------------------------------------- # +def is_minified(data: Optional[bytes]) -> bool: + """Heuristic for "we cannot show a useful line diff of this file". + + True when the bytes do not decode as UTF-8, or when any of the line-shape + thresholds in the design spec are met. + """ + if data is None: + return False + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return True + + lines = text.splitlines() or [text] + line_count = len(lines) + longest = max((len(line) for line in lines), default=0) + total_len = len(text) + avg = total_len / line_count if line_count else float(total_len) + + if longest > _MINIFIED_LONGEST_LINE: + return True + if avg > _MINIFIED_AVG_LINE: + return True + if line_count < _MINIFIED_DENSE_MAX_LINES and avg > _MINIFIED_DENSE_AVG: + return True + return False + + +# --------------------------------------------------------------------------- # +# unified diff +# --------------------------------------------------------------------------- # +def make_unified_diff( + old_text: str, + new_text: str, + path: str, + *, + max_lines: int = MAX_DIFF_LINES, + max_bytes: int = MAX_DIFF_BYTES, +) -> Tuple[str, bool]: + """Return ``(unified_diff_text, truncated)`` for two text blobs. + + Truncation keeps the profile bounded for large but technically-diffable files. + """ + import difflib + + diff_iter = difflib.unified_diff( + old_text.splitlines(), + new_text.splitlines(), + fromfile=f"a/{path}", + tofile=f"b/{path}", + lineterm="", + ) + + out: List[str] = [] + truncated = False + total = 0 + for i, line in enumerate(diff_iter): + if i >= max_lines or total >= max_bytes: + truncated = True + break + out.append(line) + total += len(line) + 1 + return "\n".join(out), truncated + + +# --------------------------------------------------------------------------- # +# manifest normalization +# --------------------------------------------------------------------------- # +def _is_host_pattern(value: Any) -> bool: + if not isinstance(value, str): + return False + return value == "" or "://" in value + + +def normalize_manifest_state(manifest: Dict[str, Any]) -> Dict[str, Any]: + """Pull the objective, comparable fields out of a manifest. + + For MV2 the host match patterns live inside ``permissions`` (and + ``optional_permissions``); we split them out into ``host_permissions`` so + permission diffs compare API permissions and host grants separately, exactly + like MV3. + """ + manifest = manifest if isinstance(manifest, dict) else {} + mv = manifest.get("manifest_version") + + permissions = list(manifest.get("permissions") or []) + optional = list(manifest.get("optional_permissions") or []) + host_perms = list(manifest.get("host_permissions") or []) + + if mv == 2: + host_perms += [p for p in permissions if _is_host_pattern(p)] + host_perms += [p for p in optional if _is_host_pattern(p)] + permissions = [p for p in permissions if not _is_host_pattern(p)] + optional = [p for p in optional if not _is_host_pattern(p)] + + return { + "manifest_version": mv if isinstance(mv, int) else None, + "permissions": _sorted_unique(permissions), + "optional_permissions": _sorted_unique(optional), + "host_permissions": _sorted_unique(host_perms), + "content_scripts": manifest.get("content_scripts"), + "background": manifest.get("background"), + "content_security_policy": manifest.get("content_security_policy"), + "web_accessible_resources": manifest.get("web_accessible_resources"), + } + + +# --------------------------------------------------------------------------- # +# reading an extension (zip or directory) +# --------------------------------------------------------------------------- # +def _read_files(source: Union[str, Path]) -> List[Tuple[str, bytes]]: + path = Path(source) + files: List[Tuple[str, bytes]] = [] + + if zipfile.is_zipfile(path): + with zipfile.ZipFile(path) as zf: + for info in zf.infolist(): + if info.is_dir(): + continue + files.append((_norm_path(info.filename), zf.read(info))) + elif path.is_dir(): + for fp in sorted(path.rglob("*")): + if fp.is_file(): + files.append((_norm_path(fp.relative_to(path).as_posix()), fp.read_bytes())) + else: + raise ValueError(f"not a zip archive or directory: {source}") + + return files + + +def _extract_manifest_and_reroot( + files: List[Tuple[str, bytes]], +) -> Tuple[Dict[str, Any], List[Tuple[str, bytes]]]: + """Locate manifest.json and re-root every path relative to its directory. + + Handles archives that wrap the extension in a top-level folder. + """ + candidates = [f for f in files if f[0].rsplit("/", 1)[-1] == "manifest.json"] + if not candidates: + raise ValueError("manifest.json not found in extension") + + manifest_path, manifest_bytes = min(candidates, key=lambda f: f[0].count("/")) + manifest = json.loads(manifest_bytes.decode("utf-8")) + + root = manifest_path.rsplit("/", 1)[0] + "/" if "/" in manifest_path else "" + rerooted: List[Tuple[str, bytes]] = [] + for path, data in files: + if root and not path.startswith(root): + continue + rerooted.append((path[len(root):], data)) + return manifest, rerooted + + +# --------------------------------------------------------------------------- # +# snapshot +# --------------------------------------------------------------------------- # +def _normalize_verdict(verdict: Dict[str, Any]) -> Dict[str, Any]: + return { + "risk_grade": verdict.get("risk_grade"), + "result_id": verdict.get("result_id"), + "analyzed_at": verdict.get("analyzed_at"), + } + + +def build_snapshot( + source: Union[str, Path], + *, + verdict: Optional[Dict[str, Any]] = None, + captured_at: Optional[str] = None, +) -> Tuple[Snapshot, Dict[str, bytes]]: + """Build an objective snapshot from an extension archive or extracted dir. + + Returns ``(snapshot, file_bytes)``. ``file_bytes`` maps path -> raw bytes for + the *current* version; pass it to :func:`compute_diff` (and/or upload it to + the blob store) so the next version can produce inline diffs. + """ + raw_files = _read_files(source) + manifest, files = _extract_manifest_and_reroot(raw_files) + + file_entries: List[Dict[str, Any]] = [] + file_bytes: Dict[str, bytes] = {} + for path, data in files: + file_entries.append({"path": path, "sha256": _sha256_bytes(data), "size": len(data)}) + file_bytes[path] = data + file_entries.sort(key=lambda e: e["path"]) + + snapshot: Snapshot = { + "version": str(manifest.get("version", "")), + "captured_at": captured_at or _now_iso(), + "content_hash": content_hash(file_entries), + **normalize_manifest_state(manifest), + "files": file_entries, + } + if verdict is not None: + snapshot["verdict"] = _normalize_verdict(verdict) + + return snapshot, file_bytes + + +# --------------------------------------------------------------------------- # +# diff +# --------------------------------------------------------------------------- # +def _string_set_delta(prev: Any, curr: Any) -> Dict[str, List[str]]: + prev_set = set(prev or []) + curr_set = set(curr or []) + return { + "added": sorted(curr_set - prev_set), + "removed": sorted(prev_set - curr_set), + } + + +_MANIFEST_DIFF_FIELDS = ( + "manifest_version", + "content_scripts", + "content_security_policy", + "web_accessible_resources", +) + + +def _manifest_changes(prev: Snapshot, curr: Snapshot) -> List[Dict[str, Any]]: + changes: List[Dict[str, Any]] = [] + for field in _MANIFEST_DIFF_FIELDS: + before, after = prev.get(field), curr.get(field) + if before != after: + changes.append({"field": field, "from": before, "to": after}) + + # background gets sub-field granularity (e.g. background.service_worker). + prev_bg = prev.get("background") or {} + curr_bg = curr.get("background") or {} + if isinstance(prev_bg, dict) and isinstance(curr_bg, dict): + for key in sorted(set(prev_bg) | set(curr_bg)): + if prev_bg.get(key) != curr_bg.get(key): + changes.append({ + "field": f"background.{key}", + "from": prev_bg.get(key), + "to": curr_bg.get(key), + }) + elif prev_bg != curr_bg: + changes.append({"field": "background", "from": prev.get("background"), "to": curr.get("background")}) + + return changes + + +def _build_modified_entry( + path: str, + from_sha: str, + to_sha: str, + curr_file_bytes: Optional[Dict[str, bytes]], + blob_loader: Optional[BlobLoader], + max_diff_lines: int, + max_diff_bytes: int, +) -> Dict[str, Any]: + entry: Dict[str, Any] = { + "path": path, + "from_sha256": from_sha, + "to_sha256": to_sha, + "blob_ref": {"from": f"nexus://blobs/{from_sha}", "to": f"nexus://blobs/{to_sha}"}, + "diff_format": "unified", + "diff": None, + "diff_truncated": False, + "is_minified": False, + } + + old_bytes = blob_loader(from_sha) if blob_loader else None + new_bytes = curr_file_bytes.get(path) if curr_file_bytes else None + + # Pointer-only when we can't see both sides (no blob upload / no current bytes). + if old_bytes is None or new_bytes is None: + return entry + + if is_minified(old_bytes) or is_minified(new_bytes): + entry["is_minified"] = True + return entry # diff stays null; blob_ref carries the pointers + + diff_text, truncated = make_unified_diff( + old_bytes.decode("utf-8"), + new_bytes.decode("utf-8"), + path, + max_lines=max_diff_lines, + max_bytes=max_diff_bytes, + ) + entry["diff"] = diff_text + entry["diff_truncated"] = truncated + return entry + + +def _file_diff( + prev: Snapshot, + curr: Snapshot, + curr_file_bytes: Optional[Dict[str, bytes]], + blob_loader: Optional[BlobLoader], + max_diff_lines: int, + max_diff_bytes: int, +) -> Dict[str, Any]: + prev_by_path = {f["path"]: f for f in prev.get("files", [])} + curr_by_path = {f["path"]: f for f in curr.get("files", [])} + prev_paths = set(prev_by_path) + curr_paths = set(curr_by_path) + + modified: List[Dict[str, Any]] = [] + for path in sorted(prev_paths & curr_paths): + from_sha = prev_by_path[path]["sha256"] + to_sha = curr_by_path[path]["sha256"] + if from_sha == to_sha: + continue + modified.append(_build_modified_entry( + path, from_sha, to_sha, curr_file_bytes, blob_loader, + max_diff_lines, max_diff_bytes, + )) + + return { + "added": sorted(curr_paths - prev_paths), + "removed": sorted(prev_paths - curr_paths), + "modified": modified, + } + + +def compute_diff( + prev_snapshot: Snapshot, + curr_snapshot: Snapshot, + *, + curr_file_bytes: Optional[Dict[str, bytes]] = None, + blob_loader: Optional[BlobLoader] = None, + max_diff_lines: int = MAX_DIFF_LINES, + max_diff_bytes: int = MAX_DIFF_BYTES, +) -> Dict[str, Any]: + """Diff two snapshots: permission deltas, manifest changes, file changes. + + Inline file diffs need both sides' bytes: the current version comes from + ``curr_file_bytes`` and the previous version from ``blob_loader(sha256)``. + When either is unavailable for a file, that file degrades to pointer-only + (``blob_ref`` set, ``diff=null``). Minified files are pointer-only by design. + """ + return { + "previous_version": prev_snapshot.get("version"), + "permissions": _string_set_delta(prev_snapshot.get("permissions"), curr_snapshot.get("permissions")), + "optional_permissions": _string_set_delta( + prev_snapshot.get("optional_permissions"), curr_snapshot.get("optional_permissions") + ), + "host_permissions": _string_set_delta( + prev_snapshot.get("host_permissions"), curr_snapshot.get("host_permissions") + ), + "manifest_changes": _manifest_changes(prev_snapshot, curr_snapshot), + "files": _file_diff( + prev_snapshot, curr_snapshot, curr_file_bytes, blob_loader, + max_diff_lines, max_diff_bytes, + ), + } + + +# --------------------------------------------------------------------------- # +# profile assembly +# --------------------------------------------------------------------------- # +def build_profile( + curr_snapshot: Snapshot, + prev_profile: Optional[Profile] = None, + *, + ext_id: Optional[str] = None, + browser: str = "chrome", + ext_name: Optional[str] = None, + publisher: Optional[str] = None, + curr_file_bytes: Optional[Dict[str, bytes]] = None, + blob_loader: Optional[BlobLoader] = None, +) -> Profile: + """Create a new profile or append ``curr_snapshot`` to an existing one. + + On the first version (``prev_profile is None``) the snapshot's + ``diff_from_previous`` is ``null`` and ``ext_id`` must be supplied. On later + versions the diff against the latest stored snapshot is attached and + ``ext_id``/identity carry over from ``prev_profile``. + """ + if prev_profile is None: + if not ext_id: + raise ValueError("ext_id is required when creating a new profile") + first_snapshot = {**curr_snapshot, "diff_from_previous": None} + return { + "schema_version": SCHEMA_VERSION, + "ext_id": ext_id, + "browser": browser, + "ext_name": ext_name, + "publisher": publisher, + "first_seen": first_snapshot["captured_at"], + "last_updated": first_snapshot["captured_at"], + "latest_version": first_snapshot["version"], + "snapshots": [first_snapshot], + } + + prev_snapshots = prev_profile.get("snapshots") or [] + if not prev_snapshots: + raise ValueError("prev_profile has no snapshots to diff against") + prev_snapshot = prev_snapshots[-1] + + diff = compute_diff( + prev_snapshot, curr_snapshot, + curr_file_bytes=curr_file_bytes, blob_loader=blob_loader, + ) + new_snapshot = {**curr_snapshot, "diff_from_previous": diff} + + profile = dict(prev_profile) + profile["snapshots"] = list(prev_snapshots) + [new_snapshot] + profile["last_updated"] = new_snapshot["captured_at"] + profile["latest_version"] = new_snapshot["version"] + if ext_name is not None: + profile["ext_name"] = ext_name + if publisher is not None: + profile["publisher"] = publisher + return profile + + +# --------------------------------------------------------------------------- # +# validation +# --------------------------------------------------------------------------- # +def _load_schema() -> Dict[str, Any]: + global _SCHEMA_CACHE + if _SCHEMA_CACHE is None: + schema_path = Path(__file__).with_name("extension-profile.schema.json") + _SCHEMA_CACHE = json.loads(schema_path.read_text(encoding="utf-8")) + return _SCHEMA_CACHE + + +def validate_profile(profile: Dict[str, Any]) -> List[str]: + """Validate a profile against the schema. Returns a list of error strings; + an empty list means the profile is valid.""" + import jsonschema + + schema = _load_schema() + validator = jsonschema.Draft202012Validator(schema) + errors = sorted(validator.iter_errors(profile), key=lambda e: list(e.path)) + return [f"{'/'.join(str(p) for p in e.path) or ''}: {e.message}" for e in errors] diff --git a/backend/profile/extension-profile.schema.json b/backend/profile/extension-profile.schema.json new file mode 100644 index 00000000..6a0ea4ef --- /dev/null +++ b/backend/profile/extension-profile.schema.json @@ -0,0 +1,131 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://suppressor/extension-profile.schema.json", + "title": "Extension Profile", + "description": "Objective, version-by-version change history of a browser extension. Not an analysis-result store: only manifest facts, file hashes/sizes, and a thin verdict breadcrumb are kept.", + "type": "object", + "required": ["schema_version", "ext_id", "browser", "latest_version", "snapshots"], + "additionalProperties": true, + "properties": { + "schema_version": {"type": "string"}, + "ext_id": {"type": "string"}, + "browser": {"type": "string"}, + "ext_name": {"type": ["string", "null"]}, + "publisher": {"type": ["string", "null"]}, + "first_seen": {"type": "string"}, + "last_updated": {"type": "string"}, + "latest_version": {"type": "string"}, + "snapshots": { + "type": "array", + "minItems": 1, + "items": {"$ref": "#/$defs/snapshot"} + } + }, + "$defs": { + "snapshot": { + "type": "object", + "required": ["version", "captured_at", "content_hash", "manifest_version", "permissions", "host_permissions", "files"], + "additionalProperties": true, + "properties": { + "version": {"type": "string"}, + "captured_at": {"type": "string"}, + "content_hash": {"type": "string"}, + "manifest_version": {"type": ["integer", "null"]}, + "permissions": {"type": "array", "items": {"type": "string"}}, + "optional_permissions": {"type": "array", "items": {"type": "string"}}, + "host_permissions": {"type": "array", "items": {"type": "string"}}, + "content_scripts": {}, + "background": {}, + "content_security_policy": {}, + "web_accessible_resources": {}, + "files": {"type": "array", "items": {"$ref": "#/$defs/file"}}, + "verdict": {"$ref": "#/$defs/verdict"}, + "diff_from_previous": {"oneOf": [{"type": "null"}, {"$ref": "#/$defs/diff"}]} + } + }, + "file": { + "type": "object", + "required": ["path", "sha256", "size"], + "additionalProperties": true, + "properties": { + "path": {"type": "string"}, + "sha256": {"type": "string"}, + "size": {"type": "integer", "minimum": 0} + } + }, + "verdict": { + "type": "object", + "description": "Breadcrumb only: how this version was graded at capture time. Full result lives in the analysis-result store.", + "additionalProperties": true, + "properties": { + "risk_grade": {"type": ["string", "null"]}, + "result_id": {"type": ["string", "null"]}, + "analyzed_at": {"type": ["string", "null"]} + } + }, + "diff": { + "type": "object", + "required": ["files"], + "additionalProperties": true, + "properties": { + "previous_version": {"type": ["string", "null"]}, + "permissions": {"$ref": "#/$defs/set_delta"}, + "optional_permissions": {"$ref": "#/$defs/set_delta"}, + "host_permissions": {"$ref": "#/$defs/set_delta"}, + "manifest_changes": {"type": "array", "items": {"$ref": "#/$defs/manifest_change"}}, + "files": {"$ref": "#/$defs/file_diff"} + } + }, + "set_delta": { + "type": "object", + "required": ["added", "removed"], + "additionalProperties": false, + "properties": { + "added": {"type": "array", "items": {"type": "string"}}, + "removed": {"type": "array", "items": {"type": "string"}} + } + }, + "manifest_change": { + "type": "object", + "required": ["field", "from", "to"], + "additionalProperties": false, + "properties": { + "field": {"type": "string"}, + "from": {}, + "to": {} + } + }, + "file_diff": { + "type": "object", + "required": ["added", "removed", "modified"], + "additionalProperties": false, + "properties": { + "added": {"type": "array", "items": {"type": "string"}}, + "removed": {"type": "array", "items": {"type": "string"}}, + "modified": {"type": "array", "items": {"$ref": "#/$defs/modified_file"}} + } + }, + "modified_file": { + "type": "object", + "required": ["path", "from_sha256", "to_sha256", "blob_ref", "is_minified", "diff_format", "diff", "diff_truncated"], + "additionalProperties": false, + "properties": { + "path": {"type": "string"}, + "from_sha256": {"type": "string"}, + "to_sha256": {"type": "string"}, + "blob_ref": { + "type": "object", + "additionalProperties": false, + "properties": { + "from": {"type": ["string", "null"]}, + "to": {"type": ["string", "null"]} + } + }, + "is_minified": {"type": "boolean"}, + "diff_format": {"type": "string"}, + "diff": {"type": ["string", "null"]}, + "diff_truncated": {"type": "boolean"} + } + } + } +} diff --git a/backend/profile/local_store.py b/backend/profile/local_store.py new file mode 100644 index 00000000..e935507b --- /dev/null +++ b/backend/profile/local_store.py @@ -0,0 +1,78 @@ +"""Local filesystem store for Extension Profiles. + +A stand-in for the eventual Supabase + Nexus backend so the profile step can run +end-to-end without external services. Layout under ``PROFILE_STORE_DIR``:: + + profiles/.json one accumulating profile document per extension + blobs/ raw file bytes, content-addressed (dedup) + +The blob directory lets the *next* version produce inline diffs: each scan stores +the current version's file bytes here, and ``make_blob_loader`` reads previous +versions back by sha256. Swap this module out for Nexus/Supabase later without +touching ``builder``. +""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +from typing import Callable, Dict, Optional + + +def _store_dir() -> Path: + return Path(os.getenv("PROFILE_STORE_DIR", "./profiles")) + + +def _profiles_dir() -> Path: + return _store_dir() / "profiles" + + +def _blobs_dir() -> Path: + return _store_dir() / "blobs" + + +def _safe_name(ext_id: str) -> str: + """Filesystem-safe filename for an ext_id (which may contain spaces/unicode).""" + cleaned = "".join(c if (c.isalnum() or c in "-_.") else "_" for c in ext_id.strip()) + return cleaned or "_unknown" + + +def load_profile(ext_id: str) -> Optional[dict]: + path = _profiles_dir() / f"{_safe_name(ext_id)}.json" + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return None + + +def save_profile(ext_id: str, profile: dict) -> Path: + directory = _profiles_dir() + directory.mkdir(parents=True, exist_ok=True) + path = directory / f"{_safe_name(ext_id)}.json" + path.write_text(json.dumps(profile, ensure_ascii=False, indent=2), encoding="utf-8") + return path + + +def store_blobs(file_bytes: Dict[str, bytes]) -> int: + """Content-address every file's bytes into the blob dir. Returns count newly written.""" + directory = _blobs_dir() + directory.mkdir(parents=True, exist_ok=True) + written = 0 + for data in file_bytes.values(): + sha = hashlib.sha256(data).hexdigest() + blob_path = directory / sha + if not blob_path.exists(): + blob_path.write_bytes(data) + written += 1 + return written + + +def make_blob_loader() -> Callable[[str], Optional[bytes]]: + directory = _blobs_dir() + + def _loader(sha256: str) -> Optional[bytes]: + blob_path = directory / sha256 + return blob_path.read_bytes() if blob_path.exists() else None + + return _loader diff --git a/backend/tests/profile/__init__.py b/backend/tests/profile/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/tests/profile/conftest.py b/backend/tests/profile/conftest.py new file mode 100644 index 00000000..0748b573 --- /dev/null +++ b/backend/tests/profile/conftest.py @@ -0,0 +1,7 @@ +import os +import sys + +# backend/ 를 import 루트로 추가 (profile.* import 가 stdlib profile 보다 우선) +BACKEND_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +if BACKEND_DIR not in sys.path: + sys.path.insert(0, BACKEND_DIR) diff --git a/backend/tests/profile/test_builder.py b/backend/tests/profile/test_builder.py new file mode 100644 index 00000000..7c8dfe94 --- /dev/null +++ b/backend/tests/profile/test_builder.py @@ -0,0 +1,286 @@ +"""Extension Profile JSON 생성기 단위 테스트.""" + +import io +import json +import zipfile + +import pytest + +from profile.builder import ( + build_profile, + build_snapshot, + compute_diff, + content_hash, + is_minified, + make_unified_diff, + normalize_manifest_state, + validate_profile, +) + + +# --------------------------------------------------------------------------- # +# fixtures: build chrome extension zips on the fly +# --------------------------------------------------------------------------- # +def _make_zip(tmp_path, name, manifest, files, top_dir=None): + """Write a .zip extension. ``files`` maps path -> str|bytes content.""" + path = tmp_path / name + buf = io.BytesIO() + prefix = f"{top_dir}/" if top_dir else "" + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr(prefix + "manifest.json", json.dumps(manifest)) + for fp, content in files.items(): + if isinstance(content, str): + content = content.encode("utf-8") + zf.writestr(prefix + fp, content) + path.write_bytes(buf.getvalue()) + return path + + +MV3_V1 = { + "manifest_version": 3, + "name": "Demo", + "version": "1.0", + "permissions": ["storage"], + "host_permissions": ["https://example.com/*"], + "background": {"service_worker": "old.js"}, +} + +MV3_V2 = { + "manifest_version": 3, + "name": "Demo", + "version": "1.1", + "permissions": ["storage", "tabs"], + "host_permissions": ["https://example.com/*"], + "background": {"service_worker": "new.js"}, +} + + +# --------------------------------------------------------------------------- # +# content_hash +# --------------------------------------------------------------------------- # +def test_content_hash_is_order_independent(): + a = [{"path": "a.js", "sha256": "x"}, {"path": "b.js", "sha256": "y"}] + b = [{"path": "b.js", "sha256": "y"}, {"path": "a.js", "sha256": "x"}] + assert content_hash(a) == content_hash(b) + assert content_hash(a).startswith("sha256:") + + +def test_content_hash_changes_with_content(): + a = [{"path": "a.js", "sha256": "x"}] + b = [{"path": "a.js", "sha256": "z"}] + assert content_hash(a) != content_hash(b) + + +# --------------------------------------------------------------------------- # +# is_minified +# --------------------------------------------------------------------------- # +def test_minified_long_single_line(): + assert is_minified(("var x=" + "a" * 6000 + ";").encode()) is True + + +def test_minified_normal_code_false(): + src = "\n".join(f"const v{i} = {i};" for i in range(50)) + assert is_minified(src.encode()) is False + + +def test_minified_undecodable_bytes_true(): + assert is_minified(b"\xff\xfe\x00\x01\x02binary") is True + + +def test_minified_none_false(): + assert is_minified(None) is False + + +# --------------------------------------------------------------------------- # +# unified diff +# --------------------------------------------------------------------------- # +def test_unified_diff_basic(): + diff, truncated = make_unified_diff("a\nb\nc", "a\nB\nc", "f.js") + assert "-b" in diff and "+B" in diff + assert truncated is False + + +def test_unified_diff_truncates(): + old = "\n".join(f"old-line-{i}" for i in range(5000)) + new = "\n".join(f"new-line-{i}" for i in range(5000)) + diff, truncated = make_unified_diff(old, new, "f.js", max_lines=50) + assert truncated is True + assert diff.count("\n") <= 50 + + +# --------------------------------------------------------------------------- # +# manifest normalization (MV2 host extraction) +# --------------------------------------------------------------------------- # +def test_mv2_hosts_split_out_of_permissions(): + state = normalize_manifest_state({ + "manifest_version": 2, + "permissions": ["tabs", "https://*.example.com/*", ""], + }) + assert state["permissions"] == ["tabs"] + assert "https://*.example.com/*" in state["host_permissions"] + assert "" in state["host_permissions"] + + +def test_mv3_permissions_unchanged(): + state = normalize_manifest_state(MV3_V1) + assert state["permissions"] == ["storage"] + assert state["host_permissions"] == ["https://example.com/*"] + + +# --------------------------------------------------------------------------- # +# build_snapshot +# --------------------------------------------------------------------------- # +def test_build_snapshot_basic(tmp_path): + z = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "console.log(1)\n"}) + snap, file_bytes = build_snapshot(z, verdict={"risk_grade": "LOW", "result_id": "r1"}) + + assert snap["version"] == "1.0" + assert snap["manifest_version"] == 3 + assert snap["permissions"] == ["storage"] + assert snap["verdict"] == {"risk_grade": "LOW", "result_id": "r1", "analyzed_at": None} + paths = {f["path"] for f in snap["files"]} + assert paths == {"manifest.json", "old.js"} + assert snap["content_hash"].startswith("sha256:") + assert "old.js" in file_bytes + + +def test_build_snapshot_reroots_top_dir(tmp_path): + z = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "x\n"}, top_dir="demo-1.0") + snap, _ = build_snapshot(z) + paths = {f["path"] for f in snap["files"]} + assert paths == {"manifest.json", "old.js"} # top dir stripped + + +def test_build_snapshot_no_manifest_raises(tmp_path): + path = tmp_path / "bad.zip" + buf = io.BytesIO() + with zipfile.ZipFile(buf, "w") as zf: + zf.writestr("readme.txt", "hi") + path.write_bytes(buf.getvalue()) + with pytest.raises(ValueError): + build_snapshot(path) + + +# --------------------------------------------------------------------------- # +# compute_diff +# --------------------------------------------------------------------------- # +def test_compute_diff_permissions_and_manifest(tmp_path): + z1 = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", MV3_V2, {"new.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, _ = build_snapshot(z2) + + diff = compute_diff(s1, s2) + assert diff["permissions"] == {"added": ["tabs"], "removed": []} + fields = {c["field"] for c in diff["manifest_changes"]} + assert "background.service_worker" in fields + sw = next(c for c in diff["manifest_changes"] if c["field"] == "background.service_worker") + assert sw["from"] == "old.js" and sw["to"] == "new.js" + + +def test_compute_diff_file_add_remove(tmp_path): + z1 = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", MV3_V2, {"new.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, _ = build_snapshot(z2) + + files = compute_diff(s1, s2)["files"] + assert "new.js" in files["added"] + assert "old.js" in files["removed"] + + +def test_compute_diff_inline_modified_with_blob_loader(tmp_path): + m1 = dict(MV3_V1) + m2 = dict(MV3_V1) + m2["version"] = "1.1" + old_src = "line1\nline2\nline3\n" + new_src = "line1\nCHANGED\nline3\n" + z1 = _make_zip(tmp_path, "v1.zip", m1, {"app.js": old_src}) + z2 = _make_zip(tmp_path, "v2.zip", m2, {"app.js": new_src}) + + s1, bytes1 = build_snapshot(z1) + s2, bytes2 = build_snapshot(z2) + + # blob_loader resolves previous-version bytes by sha256 (stand-in for Nexus). + store = {f["sha256"]: bytes1[f["path"]] for f in s1["files"]} + diff = compute_diff(s2_prev := s1, s2, curr_file_bytes=bytes2, + blob_loader=lambda sha: store.get(sha)) + + mod = next(m for m in diff["files"]["modified"] if m["path"] == "app.js") + assert mod["is_minified"] is False + assert mod["diff"] is not None + assert "-line2" in mod["diff"] and "+CHANGED" in mod["diff"] + assert mod["blob_ref"]["from"].startswith("nexus://blobs/") + + +def test_compute_diff_modified_pointer_only_without_blob(tmp_path): + m1 = dict(MV3_V1) + m2 = dict(MV3_V1) + m2["version"] = "1.1" + z1 = _make_zip(tmp_path, "v1.zip", m1, {"app.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", m2, {"app.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, b2 = build_snapshot(z2) + + diff = compute_diff(s1, s2, curr_file_bytes=b2) # no blob_loader -> no prev bytes + mod = next(m for m in diff["files"]["modified"] if m["path"] == "app.js") + assert mod["diff"] is None + assert mod["blob_ref"]["from"].startswith("nexus://blobs/") + + +def test_compute_diff_minified_modified_no_inline(tmp_path): + m1 = dict(MV3_V1) + m2 = dict(MV3_V1) + m2["version"] = "1.1" + old_min = "var a=" + "1" * 6000 + ";" + new_min = "var a=" + "2" * 6000 + ";" + z1 = _make_zip(tmp_path, "v1.zip", m1, {"min.js": old_min}) + z2 = _make_zip(tmp_path, "v2.zip", m2, {"min.js": new_min}) + s1, b1 = build_snapshot(z1) + s2, b2 = build_snapshot(z2) + + store = {f["sha256"]: b1[f["path"]] for f in s1["files"]} + diff = compute_diff(s1, s2, curr_file_bytes=b2, blob_loader=lambda sha: store.get(sha)) + mod = next(m for m in diff["files"]["modified"] if m["path"] == "min.js") + assert mod["is_minified"] is True + assert mod["diff"] is None + + +# --------------------------------------------------------------------------- # +# build_profile + validate_profile +# --------------------------------------------------------------------------- # +def test_build_profile_first_version_validates(tmp_path): + z = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + snap, _ = build_snapshot(z, verdict={"risk_grade": "LOW", "result_id": "r1"}) + profile = build_profile(snap, ext_id="abc123", ext_name="Demo") + + assert profile["latest_version"] == "1.0" + assert profile["snapshots"][0]["diff_from_previous"] is None + assert validate_profile(profile) == [] + + +def test_build_profile_second_version_attaches_diff(tmp_path): + z1 = _make_zip(tmp_path, "v1.zip", MV3_V1, {"old.js": "a\n"}) + z2 = _make_zip(tmp_path, "v2.zip", MV3_V2, {"new.js": "b\n"}) + s1, _ = build_snapshot(z1) + s2, b2 = build_snapshot(z2) + + p1 = build_profile(s1, ext_id="abc123") + p2 = build_profile(s2, p1, curr_file_bytes=b2) + + assert len(p2["snapshots"]) == 2 + assert p2["latest_version"] == "1.1" + last = p2["snapshots"][-1]["diff_from_previous"] + assert last["previous_version"] == "1.0" + assert last["permissions"]["added"] == ["tabs"] + assert validate_profile(p2) == [] + + +def test_build_profile_requires_ext_id_for_new(): + with pytest.raises(ValueError): + build_profile({"version": "1.0", "captured_at": "t", "content_hash": "h", "files": []}) + + +def test_validate_profile_reports_errors(): + errors = validate_profile({"schema_version": "1.0"}) # missing required fields + assert errors # non-empty -> invalid diff --git a/embedding/embedding.json b/embedding/embedding.json index ed9b8403..1630249e 100644 --- a/embedding/embedding.json +++ b/embedding/embedding.json @@ -1,1026 +1,1026 @@ [ - -0.03485036, - -0.006523965, - -0.024870101, - -0.021951621, - -0.04676112, - 0.0019194462, - 0.024405016, - 0.035693713, - -0.008592627, - 0.026169837, - -0.012792799, - -0.01111802, - -0.020802893, - -0.013956348, - -0.0027344371, - 1.18717335e-05, - 0.016262613, - 0.02039375, - -0.010071774, - -0.01875823, - -0.018803325, - -0.009060597, - 0.013147215, - 0.028142855, - -0.04177979, - 0.02827999, - -0.012310784, - -0.06533352, - -0.025828896, - 0.04129906, - -0.003782613, - -0.022862246, - 0.033427875, - -0.0054732924, - -0.038241345, - -0.029082129, - -0.013977575, - -0.01715362, - -0.08533639, - 0.00830145, - -0.03488972, - 0.010252315, - -0.005535598, - -0.017507868, - 0.0075469627, - -0.03480188, - -0.000507779, - -0.023278741, - -0.020672912, - -0.06980631, - -0.023200769, - -0.004430873, - 0.011973691, - -0.017140994, - 0.044351693, - 0.043278854, - -0.042956855, - -0.014997362, - -0.043559704, - 0.023506247, - -0.053146906, - -0.010448102, - -0.020760927, - 0.013700241, - 0.0028324132, - 0.030927282, - 0.040706042, - -0.00079867116, - -0.0016535603, - -0.029102279, - -0.0036018426, - 0.0068588485, - -0.0143683525, - -0.017418599, - -0.06759086, - 0.04024998, - 0.0013544558, - -0.027406689, - -0.0076633324, - 0.028525572, - -0.040828053, - -0.006313824, - 0.055598367, - 0.0038820985, - -0.0057513574, - 0.029751923, - -0.02152409, - 0.045518126, - 0.056295734, - -0.014960843, - -0.023993399, - -0.03368726, - 0.05779403, - -0.046619095, - -0.03961416, - 0.004086139, - 0.002741565, - -0.028147021, - 0.06516865, - 0.016405731, - -0.010616796, - 0.018252261, - 0.005795998, - -0.0053562457, - 0.04854147, - 0.04298808, - 0.019046724, - -0.0052411426, - -0.004604102, - -0.011814224, - -0.009750135, - 0.015850324, - 0.03265538, - 0.020712115, - -0.006471794, - -0.04911774, - 0.01483612, - -0.01579769, - -0.013883977, - -0.015874514, - 0.03148097, - 0.04799249, - 0.053061053, - -0.037908312, - 0.04927149, - 0.0094831465, - 0.027591636, - 0.03283308, - -0.0038896052, - 0.033055276, - 0.027255742, - 0.03658406, - -0.015679047, - -0.0034444723, - -0.08805762, - -0.006555244, - -0.0034031516, - 0.035184074, - 0.04333371, - -0.0530124, - 0.003475752, - 0.023959255, - -0.0341101, - -0.026977474, - 0.0306026, - -0.04823844, - 0.005313242, - 0.04337344, - -0.016344374, - -0.052851405, - 0.025235528, - -0.008192982, - 0.042521715, - 0.03924706, - -0.017751662, - -0.06547015, - 0.015605447, - 0.0025222667, - 0.052172974, - -0.01636314, - 0.005909687, - 0.0063013383, - -0.031018585, - 0.024667436, - 0.04233253, - -0.013303014, - 0.013495011, - -0.011126833, - -0.038032837, - -0.0042188154, - 0.008843474, - -0.037746347, - 0.008016975, - 0.013132684, - 0.011910548, - 0.011193239, - 0.060552154, - 0.01644624, - -0.011260529, - -0.028127898, - -0.033737317, - -0.009595608, - -0.012269939, - -0.017357165, - -0.021629686, - 0.017643092, - -0.029173048, - 0.0032606227, - -0.008527376, - 0.05322887, - 0.013322519, - -0.010476832, - 0.03258906, - 0.03202706, - -0.03405511, - -0.018965025, - 0.026686324, - -0.0035002579, - 0.0034975614, - -0.027884498, - 0.022383856, - 0.034188, - 0.045019623, - -0.016051197, - -0.045693796, - -0.018393034, - -0.015090342, - -0.038736198, - 0.007350469, - -0.013768265, - 0.022838596, - 0.0048558167, - -0.007734886, - -0.022188174, - -0.023705874, - -0.016501134, - 0.0037238947, - -0.012784965, - 0.011435852, - -0.026673306, - -0.005509241, - 0.02364078, - 0.0025442347, - -0.009661722, - 0.0073032924, - 0.016039537, - 0.031440396, - 0.004281652, - -0.03770763, - 0.028144414, - -0.0010302472, - 0.031301018, - 0.03395194, - -0.0014075655, - 0.01414747, - -0.049955558, - -0.0012861338, - 0.017909067, - 0.028062966, - -0.02871389, - 0.04881674, - -0.063322306, - 0.038415723, - -0.011686032, - -0.042711668, - -0.017449714, - 0.002141979, - 0.0457172, - -0.04424886, - -0.017762464, - -0.006207608, - 0.01629242, - -0.037824757, - -0.011094415, - 0.014849885, - 0.0038221972, - 0.008234737, - 0.026768895, - -0.0046303924, - 0.01726791, - -0.014142183, - 0.019061428, - 0.01821286, - 0.024755321, - 0.027596371, - -0.011208457, - 0.0010563019, - -0.023535619, - -0.010621036, - -0.00074850395, - -0.0036844595, - -0.019430794, - 0.04973371, - -0.020635622, - -0.0047764294, - 0.019615741, - -0.018160922, - 0.017911669, - 0.08115741, - 0.026115179, - -0.006572227, - 0.012608993, - 0.045932107, - 0.0027327065, - 0.035970118, - 0.008203159, - -0.030672822, - 0.014164894, - -0.031151213, - -0.03182053, - -0.028612746, - 0.021081073, - 0.066009685, - -0.04395738, - 0.022311244, - 0.011704254, - -0.03953015, - -0.17731258, - 0.02446469, - 0.02200765, - 0.033863176, - 0.009451169, - -0.009153969, - -0.024325192, - -0.016987937, - -0.03815129, - 0.03530359, - -0.07062162, - -0.056043025, - -0.0111074345, - -0.00892553, - 0.04874841, - 0.022275368, - -0.00049478165, - -0.010223968, - -0.014345399, - -0.012954244, - -0.054299448, - -0.014125319, - 0.023162726, - 0.021651436, - -0.012387239, - 0.0051369322, - 0.031682234, - -0.019705169, - -0.0074669328, - 0.02000536, - -0.008161489, - -0.0100305565, - -0.0013780366, - 0.028171338, - -0.009415919, - 0.0021736685, - 0.010446073, - -0.010456459, - 0.009267223, - -0.02139086, - 0.043684043, - 0.07192079, - -0.007597984, - 0.02457813, - 0.015363794, - -0.025522145, - 0.0014028245, - -0.023182474, - -0.053059332, - -0.02515991, - -0.033534322, - -0.024742566, - 0.009469918, - 0.02868977, - -0.06879678, - 0.004279142, - -0.021392403, - 0.029523512, - 0.0140930265, - 0.040809836, - -0.0034976192, - -0.027779879, - 0.024514323, - -0.07213847, - -0.0006524821, - -0.02826045, - 0.048646033, - -0.00341837, - 0.011114654, - -0.031905606, - 0.019060064, - -0.05665736, - 0.02308151, - 0.017008655, - -0.002823673, - 0.011452782, - -0.03378409, - -0.009196477, - 0.0034898797, - -0.111773275, - 0.006144034, - 0.019115357, - 0.023291895, - -0.0086863665, - -0.03703144, - -0.062035933, - 0.0115044, - -0.021629985, - 0.01940764, - 0.2651562, - -0.0028407408, - -0.031331632, - 0.04184685, - 0.0192927, - -0.018956458, - -0.016081788, - 0.079190314, - -0.012182804, - -0.012182674, - 0.030636767, - 0.03426719, - 0.016827187, - -0.0012515389, - 0.02072367, - 0.02729455, - -0.044176634, - 0.001213331, - 0.06003864, - -0.010951624, - 0.011007386, - -0.0120707825, - 0.01967973, - 0.0095139835, - -0.0331325, - -0.03138036, - -0.005941679, - -0.017965306, - 0.0039845016, - 0.055597417, - -0.028543742, - 0.025939127, - 0.04222716, - -0.027559107, - -0.073364444, - 0.028254727, - 0.047389302, - 0.011136994, - -0.02329983, - -0.0014732265, - -0.032746255, - -0.011333437, - -0.050461497, - 0.009710012, - -0.0007233123, - -0.0023836573, - -0.032397136, - -0.029447773, - -0.03913593, - 0.022129782, - 0.022849508, - -0.0028128333, - 0.022123925, - -0.044084348, - 0.003719973, - -0.042123318, - -0.0056991354, - -0.00852776, - 0.00312135, - 0.023790535, - 0.011400686, - 0.011409521, - -0.05798747, - 0.011576273, - -0.010227346, - 0.005631751, - 0.007495863, - 0.00804295, - 0.076911, - 0.023302251, - 0.011465635, - 0.017791932, - -0.0011115364, - 0.01702136, - 0.0064648306, - 0.017583003, - 0.064273424, - 0.03191352, - -0.04347176, - -0.0084685655, - -0.007889204, - -0.04106867, - -0.015657114, - -0.022527304, - 0.0010747634, - -0.009583432, - -0.01001918, - 0.057714473, - 0.012642176, - 0.014514113, - 0.008418106, - 0.0029056633, - -0.02109183, - 0.08549639, - -0.02401644, - 0.008622991, - 0.009177995, - -0.027816093, - -0.03818879, - -0.012544949, - -0.048338052, - -0.059360508, - -0.026988946, - 0.012060502, - 0.00735656, - 0.01653607, - -0.0148168625, - 0.028525626, - -0.020907076, - 0.028334254, - -0.033567596, - 0.019952297, - 0.01571829, - -0.030224144, - -0.0078595085, - 0.07016056, - 0.030943854, - 0.018744482, - 0.06552349, - 0.019052256, - -0.048748452, - 0.040406156, - 0.008260294, - 0.038071383, - -0.032423597, - -0.042639453, - -0.022628836, - 0.036644712, - -0.011333548, - 0.05039947, - 0.011461258, - -0.02223915, - 0.009473547, - 0.026953608, - 0.036760084, - 0.0017486485, - 0.013157991, - 0.0059267255, - 0.007039213, - 0.03609918, - -0.018248925, - 0.018680604, - -0.02767531, - 0.03577312, - 0.00055750203, - 0.02181149, - -0.028532414, - -0.011596229, - -0.0005457933, - 0.035079267, - 0.02135478, - 0.07462979, - 0.034515195, - 0.004064192, - 0.001072951, - -0.04284846, - -0.030484488, - -0.0036550527, - 0.006420163, - 0.012750347, - -0.012521361, - -0.01923254, - 0.015273171, - 0.02544576, - -0.015367402, - 0.019597257, - 0.02435614, - 0.032175273, - -0.038116544, - -0.00093479076, - 0.0013199619, - -0.048687097, - -0.028815554, - 0.032724977, - -0.029785093, - 0.016622232, - -0.018459229, - -0.053232837, - -0.013562255, - -0.030861689, - 0.026351627, - -0.0030361135, - -0.00872005, - -0.033535004, - 0.03025441, - -0.014477477, - 0.050545838, - -0.0028624274, - 0.004125899, - -0.032477897, - -0.008121306, - 0.12731615, - 0.026881339, - -0.014786382, - 0.035495725, - 0.00082117005, - 0.04081602, - -0.013261213, - 0.0069885044, - 0.013010508, - 0.006064077, - -0.030903772, - 0.015621737, - -0.025409952, - -0.019215796, - 0.008197931, - -0.0059183026, - -0.0133599, - -0.058643367, - -0.016191332, - 0.021001328, - 0.022470305, - -0.01905571, - 0.034924906, - 0.0012492832, - -0.027229425, - 0.028659761, - 0.050299875, - 0.028339231, - -0.032533582, - 0.03901179, - -0.015456914, - -0.012930993, - -0.029640103, - -0.0027097326, - 0.0042801523, - 0.031620346, - -0.049821004, - 0.013133078, - -0.01899334, - -0.007282641, - -0.035775866, - 0.016878197, - -0.053011406, - -0.007267962, - -0.03583142, - 0.018804934, - 0.036747944, - -0.009275369, - -0.021193573, - 0.048727028, - -0.006180164, - 0.029412761, - 0.02580566, - 0.025216166, - 0.011644537, - -0.064970955, - 0.022830062, - 0.0027662478, - 0.024282526, - -0.04159977, - -0.003701238, - 0.006597354, - -0.06000885, - 0.008319213, - 0.018260457, - -0.009221352, - -0.006496549, - -0.06236123, - -0.047565293, - -0.029486923, - 0.06445119, - -0.013600725, - -0.01965479, - -0.02777698, - -0.052896906, - 0.018137082, - -0.04071823, - -0.0169595, - -0.025429947, - 0.0036767197, - 0.011745697, - 0.0048488337, - -0.00027546255, - 0.015787488, - 0.007459889, - -0.030069385, - 0.0021149854, - 0.05602186, - -0.015326911, - -0.06182895, - -0.0011165651, - 0.012864129, - -0.0016939089, - 0.0067999484, - 0.0010948898, - 0.033293694, - -0.031025292, - -0.017793514, - -0.027720565, - 0.04350979, - 0.01690967, - -0.039987262, - 0.01068626, - -0.022370197, - -0.004096849, - -0.012856249, - 0.06924249, - -0.0065072216, - -0.000597387, - 0.019755809, - 0.008910962, - -0.00568093, - -0.008516902, - -0.033760957, - 0.016812917, - -0.027316146, - -0.026889825, - 0.015981369, - 0.017211344, - -0.01262618, - -0.02413841, - -0.008612607, - 0.0055217566, - 0.055153783, - -0.035130538, - 0.020660555, - -0.049755167, - 0.024131725, - 0.020430876, - 0.023211103, - 0.03711559, - -0.019059062, - 0.0022703004, - -0.0505399, - 0.053637497, - -0.024510836, - -0.014918175, - -0.027184827, - 0.0034064827, - -0.049552877, - -0.0017316656, - 0.0044178544, - -0.040789127, - 0.0199112, - 0.015300785, - 0.017422635, - 0.010424917, - 0.03423134, - 0.00052812154, - 0.021104172, - -0.051543217, - 0.037585888, - -0.001752495, - -0.019554261, - 0.013937136, - 0.013765535, - -0.03148354, - -0.0056180186, - -0.017817277, - 0.03326038, - -0.00013004264, - -0.02274853, - -0.03417097, - -0.0066750236, - -0.0075843916, - -0.019918067, - 0.01152048, - 0.021681871, - 0.018207202, - -0.0020919684, - 0.030538194, - -0.012318519, - -0.031633995, - 0.00013115391, - -0.03362657, - 0.00907644, - 0.007918352, - 0.010078273, - 0.014963157, - -0.012318672, - -0.007231364, - -0.01027362, - 0.005270827, - -0.03693281, - 0.024665194, - -0.041493226, - -0.017185101, - 0.02431877, - -0.032249715, - -0.026663367, - -0.04970651, - -0.02164517, - 0.027656583, - -0.022452675, - -0.02569411, - -0.03141506, - 0.00958662, - -0.010624525, - 0.0013384996, - -0.017067885, - 0.010905406, - -0.02574669, - 0.027742615, - -0.16543303, - 0.0044359285, - -0.0041801147, - 0.050032545, - -0.036420442, - 0.016444901, - 0.004906219, - 0.00305755, - 0.000645408, - -0.02585195, - 5.851421e-05, - -0.0177179, - 0.017901, - -0.014389969, - -0.014035693, - 0.031114766, - -0.009225222, - 0.003178253, - -0.011938513, - 0.029872155, - -0.01868456, - -0.0047811917, - 0.04640891, - -0.028970834, - -0.003929274, - 0.037099987, - -0.03288008, - 0.025929831, - -0.017864862, - -0.001104512, - 0.029252192, - -0.03019128, - 0.021096261, - 0.039015904, - 0.020269977, - 0.055913292, - 0.029641623, - 0.016977029, - 0.037405808, - -0.0060188845, - -0.012283051, - -0.009624288, - -0.024535663, - -0.03902192, - -0.008151951, - 0.027306039, - 0.003717378, - -0.0055695516, - -0.030805841, - 0.021994106, - -0.017613834, - 0.02414003, - -0.07084169, - 0.02342745, - -0.025662372, - 0.00950205, - -0.0128408745, - 0.028426634, - -0.024287108, - 0.018353976, - -0.05276756, - 0.032767028, - -0.028602868, - -0.010929256, - -0.025840053, - 0.03306693, - -0.10701591, - 0.0071515124, - -0.004364812, - 0.017506683, - -0.026777966, - -0.025047412, - -0.012522828, - -0.003158993, - 0.01915694, - 0.0080304975, - 0.046226174, - 0.0031628187, - -0.028501768, - 0.0017846226, - 0.011096759, - -0.038458068, - -0.028518906, - 0.02648644, - 0.05748379, - -0.0052262894, - 0.00090398826, - 0.019646378, - -0.034181133, - -0.031019276, - -0.021406053, - -0.038842995, - 0.029685358, - 0.026918044, - -0.008026107, - 0.017391609, - -0.039732188, - 0.01055362, - -0.010321314, - -0.0028208157, - 0.019798337, - -0.030184537, - 0.037872516, - 0.0022543075, - -0.024099082, - 0.024197003, - -0.012363028, - -0.0036628547, - 0.0106351515, - 0.004500302, - -0.074983455, - -0.0024789374, - -0.039768178, - 0.007209211, - -0.07402429, - -0.0035831241, - 0.052372623, - 0.008074728, - -0.010254601, - 0.03723895, - -0.0337514, - -0.024283621, - -0.027402911, - -0.03691343, - 0.021020405, - 0.020925667, - 0.029383194, - 0.04252137, - 0.028472798, - -0.04295308, - 0.021773832, - -0.045806758, - -0.0066437596, - -0.0014096142, - 0.042101096, - 0.00016596373, - -0.00016136566, - 0.060364816, - -0.044781003, - -0.042765927, - -0.0654631, - -0.0038248457, - -0.010606115, - -0.05439583, - 0.024176728, - 0.022730034, - 0.03621591, - 0.0009763813, - 0.034391966, - -0.04052909, - -0.032464515, - 0.020484608, - -0.027862048, - 0.030571727, - 0.02789852, - 0.026415614, - -0.057382565, - 0.02988672, - -0.03243118, - 0.046732627, - 0.013446517, - -0.019803952, - -0.022840079, - -0.038886994, - -0.011342753, - 0.042946495, - -0.022382729, - 0.0076412205, - -0.045635633, - 0.01405105, - 0.025229704, - 0.06655055, - -0.010767731, - -0.0059707374, - 0.03609742, - -0.005530074, - 0.03376942, - 0.008308468, - 0.021039316, - -0.010723757, - 0.0355036, - -0.035310235, - 0.028243257, - 0.025039956, - 0.024774326, - 0.034099862, - 0.0011991276, - 0.017638503, - 0.026746906, - -0.019286534, - -0.0075161713, - -0.02140353, - 0.017149443, - -0.012274824, - 0.01085057, - 0.018384213, - 0.0033054748, - -0.023285326, - -0.015042774, - 0.028020097, - -0.02429766, - 0.038505014, - -0.010975393, - -0.0045849024, - 0.01914343, - -0.031177802, - 0.006690604, - -0.02940985, - -0.024704568, - -0.003650038, - 0.021816175, - 0.009830584, - -0.03013565, - 0.024328627, - -0.035891555, - -0.048526727, - 0.04881518, - -0.030377112, - 0.01964267, - 0.007051219, - 0.03593706, - 0.024328787, - 0.042422246, - -0.030494707, - -0.055859216, - -0.031119449, - 0.016314944, - -0.005308099, - -0.0041531087, - 0.0147686, - -0.04632526, - -0.01062353, - 0.0033541117, - -0.009182904, - 0.041490734, - 0.0035344528, - -0.029527742, - 0.02343805, - -0.0061483, - 2.2035414e-05, - 0.06465615, - 0.045795295, - 0.031973656, - -0.03712669 + -0.035540424, + -0.0043125497, + -0.01500155, + -0.0039612064, + -0.034220085, + -0.014673776, + 0.022932058, + 0.043721996, + 0.003023589, + -6.411021e-05, + -0.023201818, + -0.0006830563, + -0.018230671, + -0.006285979, + 0.0009594604, + -0.011561239, + 0.014386106, + 0.007685114, + 0.002560639, + -0.008862037, + -0.015705701, + 0.00026157932, + 0.0091719115, + 0.020362707, + -0.039432317, + 0.027982226, + -0.019040931, + -0.0579503, + -0.029375508, + 0.03395607, + -0.017147735, + -0.012944642, + 0.014744964, + 0.009999208, + -0.03147265, + -0.0045112236, + -0.019853005, + -0.016967569, + -0.082720324, + -0.0037163256, + -0.025519546, + 0.009049471, + 0.0071469173, + -0.033614393, + 0.009219351, + -0.03533692, + -0.011300448, + -0.026129387, + -0.0217745, + -0.061660204, + -0.018289024, + -0.013864334, + -0.0027544769, + -0.015693415, + 0.04889935, + 0.03469741, + -0.030484525, + 0.0013913357, + -0.059629466, + 0.020596897, + -0.047446992, + -0.00998671, + -0.020718874, + 0.010504385, + 0.0034249753, + 0.030279705, + 0.016118893, + -0.00080817606, + -0.0034266284, + -0.03447596, + -0.014561402, + 0.013687459, + -0.029745989, + 0.001666492, + -0.058539372, + 0.052298434, + 0.014443302, + -0.03539248, + -0.011575928, + 0.02588514, + -0.023969132, + -0.01051046, + 0.0336828, + -0.005281555, + -0.0036213396, + 0.042740744, + -0.011760336, + 0.039187573, + 0.052444637, + 0.006970902, + -0.011505029, + -0.033932637, + 0.07297291, + -0.03752474, + -0.035911724, + 0.0055007595, + 0.0018801576, + -0.012344642, + 0.031608358, + 0.0019746614, + -0.015085678, + 0.00872032, + 0.008152795, + -0.0045497543, + 0.033418335, + 0.04651675, + 0.017604962, + 0.0033722841, + 0.005495094, + -0.0070620053, + -0.014465226, + 0.02688613, + 0.04316405, + 0.017829837, + -0.0095854215, + -0.048734505, + 0.010000191, + -0.023314316, + -0.019528374, + -0.006886649, + 0.03187103, + 0.0608218, + 0.037757162, + -0.03302051, + 0.03996502, + 0.015882034, + 0.033170387, + 0.03169574, + -0.012785765, + 0.0316073, + 0.009463434, + 0.032444615, + -0.0052398057, + -0.010948876, + -0.08481821, + -0.012677158, + 0.006891006, + 0.06301468, + 0.040772405, + -0.02859976, + 0.03152613, + 0.017300107, + -0.033778034, + -0.022050386, + 0.03158338, + -0.08632693, + 0.030773796, + 0.038783383, + -0.016291793, + -0.05024555, + 0.037132595, + 0.009242082, + 0.041624844, + 0.041804973, + -0.038715784, + -0.061805855, + 0.004833871, + -0.0032721742, + 0.06482766, + -0.007136096, + -0.014284595, + 0.0050829803, + -0.027384441, + 0.037771273, + 0.03524182, + -0.012263654, + 0.0031836503, + -0.026626673, + -0.050049827, + 0.00623903, + 0.023616785, + -0.047233075, + -0.001443006, + 0.025695024, + 0.0055104177, + -0.00042990252, + 0.07034322, + 0.027812658, + -0.002557722, + -0.033141855, + -0.046404485, + -0.004359686, + -0.011891668, + -0.015275272, + -0.015015588, + 0.022325234, + -0.02437152, + 0.01250391, + -0.025172856, + 0.0352632, + 0.0076452703, + -0.004160174, + 0.028269736, + 0.016204543, + -0.039722662, + -0.031363092, + 0.040678833, + -0.0028519824, + 0.02270374, + -0.023759235, + 0.0054619815, + 0.043257996, + 0.036729176, + -0.0005154979, + -0.03816618, + -0.0277644, + -0.002231237, + -0.022062553, + -0.008970236, + -0.011127811, + 0.017840413, + 0.014614948, + 0.005539553, + -0.018068725, + -0.033656325, + -0.011433405, + 0.0024294283, + -0.008508651, + 0.005496637, + -0.025733627, + 0.0037519112, + 0.0053659026, + 0.011044733, + 0.005352398, + 0.010653699, + -0.0047489684, + 0.028495595, + 0.01162837, + -0.049981326, + 0.022937832, + -0.016780095, + 0.034425065, + 0.020969976, + -0.0061849304, + 0.030523786, + -0.05032338, + 0.007898791, + 0.020700103, + 0.03975607, + -0.020061469, + 0.05628226, + -0.067241795, + 0.04109376, + -0.011501287, + -0.03970386, + -0.011927178, + 0.010235679, + 0.04292088, + -0.050034344, + -0.018472817, + -0.018886443, + 0.007804967, + -0.04097342, + -0.012324401, + 0.013782283, + 0.009639795, + 0.0040474017, + 0.024750462, + 0.00050500676, + 0.023131112, + -0.0043665594, + 0.020119144, + 0.013851291, + 0.027416615, + 0.035738133, + -0.009420496, + -0.023711251, + -0.033187956, + 0.0067144274, + 0.0015973891, + 0.0060812645, + -0.029172974, + 0.031199431, + -0.032828603, + -0.005249675, + 0.024949718, + -0.008640782, + 0.017278261, + 0.07636377, + 0.041372664, + -0.006425257, + 0.024150407, + 0.036597442, + 0.011449838, + 0.04448599, + 0.017793471, + -0.03430268, + -0.004826711, + -0.0058154953, + -0.030509433, + -0.028473528, + 0.026003193, + 0.055330906, + -0.04789963, + 0.024928136, + 0.03194985, + -0.02493602, + -0.17091215, + 0.020887524, + 0.010800067, + 0.021790186, + 0.010625859, + -0.0074593336, + -0.02130046, + -0.031277344, + -0.038347844, + 0.049448665, + -0.052322306, + -0.054100957, + -0.028071137, + -0.02382311, + 0.041731197, + 0.02741918, + -0.014175494, + 0.0056713466, + -0.029490303, + -0.012858688, + -0.038349167, + 0.02836895, + 0.022073276, + 0.006702537, + -0.018492635, + 0.011318007, + 0.026493872, + -0.019104743, + -0.0063021244, + 0.03877698, + -0.007421214, + -0.009439214, + 0.0014386963, + 0.019966314, + -0.0110261105, + 0.0033504947, + -0.008596098, + -0.012756703, + 0.00669487, + -0.028156321, + 0.007856934, + 0.07940014, + -0.011965161, + 0.031730916, + 0.025921395, + -0.027395304, + -0.0062956195, + -0.03806656, + -0.055978604, + -0.046453428, + -0.031692877, + -0.0115980655, + -0.009553197, + 0.02696136, + -0.07207385, + 0.010008941, + 0.009161723, + 0.0367366, + 0.04008746, + 0.011571431, + 0.002059561, + -0.00578889, + 0.02206174, + -0.05897136, + -0.00039718143, + -0.01478718, + 0.05825217, + -0.0069621434, + 0.010907628, + -0.032825023, + 0.029956196, + -0.03709911, + 0.03661216, + 0.022396093, + 0.005803921, + 0.025568428, + -0.029039348, + -0.025916398, + 0.003658393, + -0.10671754, + 0.012312412, + 0.015170817, + 0.026491951, + -0.0034667796, + -0.037462182, + -0.058557436, + 0.009534287, + -0.029417442, + 0.030745089, + 0.26336142, + -0.006270348, + -0.013675315, + 0.05166998, + 0.023462333, + -0.029233214, + -0.011385997, + 0.07055414, + -0.0096172765, + -0.02137509, + 0.023463685, + 0.044137347, + 0.031541117, + 0.008113454, + 0.028550882, + 0.021089844, + -0.046045523, + 0.009343249, + 0.055118855, + 0.00059009093, + 0.010504315, + -0.01588994, + 0.024900999, + -0.017277494, + -0.039863598, + -0.031794623, + -0.010783353, + -0.014681642, + 0.0018184608, + 0.06819786, + -0.021864409, + 0.034250077, + 0.02721689, + -0.021482835, + -0.06197356, + 0.019354058, + 0.052295797, + 0.009106822, + -0.034628466, + 0.01446979, + -0.03188524, + -0.011352402, + -0.026079426, + 0.0012850903, + -0.026897343, + 0.0055958885, + -0.03781363, + -0.023671817, + -0.05045699, + 0.012651729, + 0.017164286, + -0.0129017765, + 0.01511296, + -0.04046131, + 0.017772228, + -0.019173093, + -0.000704597, + -0.0057172966, + 0.009027126, + 0.027601425, + 0.007673995, + 0.0135371275, + -0.037670046, + 0.0153490305, + -0.028624374, + -0.008729324, + 0.020493362, + -0.0038286129, + 0.04061988, + 0.037060518, + 0.017289218, + 0.032855906, + 0.003413343, + 0.018052958, + 0.011419034, + 0.0042261574, + 0.059293475, + 0.03086562, + -0.038831692, + 0.0022679062, + -0.02062474, + -0.04939548, + -0.018029746, + -0.005109428, + -0.0076940744, + 0.009497699, + -0.023193847, + 0.062326573, + 0.009280389, + 0.034507535, + -0.0065380167, + 0.010557469, + -0.015685864, + 0.08523495, + -0.013950076, + 0.029688701, + 0.028984303, + -0.03033703, + -0.038351182, + -0.023411741, + -0.03358136, + -0.03808846, + -0.024971444, + 0.010980564, + 0.018003657, + 0.02680926, + -0.012699566, + 0.03312877, + -0.030300917, + 0.014444101, + -0.0440556, + 0.01345662, + 0.02811193, + -0.015999917, + -0.015307806, + 0.08160995, + 0.031998634, + -0.0005930801, + 0.06616691, + 0.018216204, + -0.04075321, + 0.033301502, + 0.002330849, + 0.057430778, + -0.024376707, + -0.055560768, + -0.023526592, + 0.03975858, + -0.015612394, + 0.043909956, + 0.02632352, + -0.026144067, + 0.0140032545, + 0.040780038, + 0.052309003, + 0.001090255, + 0.044166446, + -0.00060999923, + 0.005574291, + 0.036202528, + -0.015676463, + 0.0021018896, + -0.013796276, + 0.048270643, + 0.002394779, + 0.035297364, + -0.04311444, + -0.0015244705, + 0.016313026, + 0.03755499, + 0.017434109, + 0.07903873, + 0.047565084, + -0.0053588343, + -0.016122881, + -0.044135395, + -0.041864943, + 0.0017292724, + 0.025358679, + 0.011261144, + -0.0037508695, + -0.0063735545, + 0.015460245, + 0.028086416, + -0.023101276, + 0.0040004905, + 0.028008131, + 0.03499609, + -0.014988292, + 0.0055555506, + -0.016675757, + -0.053690527, + -0.034661565, + 0.037158605, + -0.024301022, + 0.04268904, + -0.040901557, + -0.04794083, + -0.025262427, + -0.020267302, + 0.018436572, + -0.011063047, + -0.007838248, + -0.021223838, + 0.017515123, + -0.021018023, + 0.0381946, + 0.008762564, + 0.008467957, + -0.027179888, + -0.02167856, + 0.12485428, + 0.028081162, + -0.01125163, + 0.03168154, + -0.010894951, + 0.079535276, + -0.021919629, + 0.010740841, + 0.0042296164, + 0.0056250114, + -0.008975374, + 0.003813022, + -0.011469158, + -0.018717444, + 0.002668455, + 0.009517659, + -0.006203588, + -0.032016136, + -0.020201974, + 0.028106695, + 0.028637234, + -0.02985525, + 0.03072794, + 0.01802249, + -0.044347633, + 0.036461044, + 0.058161937, + 0.02396372, + -0.035366416, + 0.02383857, + -0.012300006, + -0.023082707, + -0.021671489, + -0.020038316, + 0.0054275077, + 0.02136522, + -0.041313924, + -0.008360628, + -0.020315383, + -0.013007147, + -0.044424202, + 0.03470291, + -0.047108315, + -0.007244148, + -0.038348764, + 0.0026577192, + 0.020647103, + 0.015472981, + -0.021628307, + 0.04248728, + -0.019476421, + 0.020273324, + 0.047409028, + 0.024564689, + 0.025159044, + -0.07095747, + 0.0010422067, + 0.004710526, + 0.01561932, + -0.040175084, + -0.0041703386, + -0.010208369, + -0.06588367, + 0.0044695903, + 0.0145629505, + -0.0031650946, + -0.0076785143, + -0.061453726, + -0.033938143, + -0.03429412, + 0.056270715, + -0.0021333904, + -0.008727723, + -0.026811391, + -0.04140576, + 0.010979562, + -0.04479837, + -0.013969982, + -0.03787035, + -0.011467582, + 0.006865206, + 0.005961738, + 0.003261076, + 0.015390911, + 0.011385705, + -0.016311577, + -0.013051688, + 0.040660527, + -0.015573549, + -0.0737897, + -0.012749576, + 0.022413855, + 0.0021245198, + -0.009327382, + 9.7007105e-05, + 0.037278116, + -0.028731985, + -0.012166975, + -0.039664686, + 0.046987552, + 0.021548439, + -0.02877028, + 0.012877035, + -0.018463923, + 0.0039465986, + -0.012279091, + 0.067423075, + -0.0038708956, + -0.010557657, + 0.013671262, + -0.0025658733, + -0.01675385, + -0.00472873, + -0.04872376, + 0.025389161, + -0.03578233, + -0.039283987, + 0.037435297, + 0.019199183, + -0.024830999, + -0.02649742, + -0.023753455, + 0.023797851, + 0.04415423, + -0.0384997, + 0.023880698, + -0.052072763, + 0.031664502, + 0.030935949, + 0.017395388, + 0.018645382, + -0.014422513, + -0.0073672016, + -0.07344133, + 0.049215563, + -0.020804355, + -0.01877317, + -0.03781439, + -0.0016161609, + -0.035837907, + -0.010174443, + 0.0010238435, + -0.028608019, + 0.016583605, + 0.0126875155, + 0.03830263, + 0.00030887304, + 0.025015088, + 0.002044907, + 0.015047575, + -0.030303938, + 0.035282847, + -0.01299968, + -0.011408893, + 0.00915301, + 0.012805768, + -0.031176593, + 0.012043452, + -0.016835786, + 0.04098914, + 0.0052500665, + -0.022866137, + -0.031467788, + -0.010642347, + -0.030589694, + -0.018889263, + 0.0014628854, + 0.013622995, + 0.034228757, + -0.0006223219, + 0.038695943, + -0.016322842, + -0.034300745, + -1.6720902e-05, + -0.0239627, + -0.0009887859, + -0.0011182632, + 0.015420228, + 0.005127973, + -0.014761954, + 0.011237161, + -0.0051204776, + 0.00057824113, + -0.03830941, + 0.027179234, + -0.038137287, + -0.01832438, + 0.03806791, + -0.013126439, + -0.03950949, + -0.067992955, + -0.0130304, + 0.015828092, + -0.008844865, + -0.026819685, + -0.04564095, + 0.0030537127, + -0.006788286, + 0.0019503232, + -0.027307421, + 0.010493203, + -0.021539237, + 0.0490111, + -0.15990233, + 0.0058901664, + -0.005970224, + 0.053682275, + -0.025622882, + 0.008835088, + -0.0066961576, + 0.003967323, + 0.015457101, + -0.008711219, + 0.012753193, + -0.012935689, + -0.002539372, + -0.02026739, + -0.023719812, + 0.032911967, + -0.020585224, + 0.012818082, + -0.027591335, + 0.022709759, + -0.010934821, + -0.015531736, + 0.042712983, + -0.028273309, + 0.014499474, + 0.03412267, + -0.011089235, + 0.0028958756, + -0.008988179, + -0.009810768, + 0.031685375, + -0.015877446, + 0.02308227, + 0.022767428, + 0.021194082, + 0.04064294, + 0.019226698, + 0.017006392, + 0.035582703, + -0.0005129924, + -0.017660828, + -0.007734722, + -0.016305344, + -0.04025412, + -0.027390027, + 0.020124434, + -0.0005577389, + -0.0069233165, + -0.022851378, + 0.004152408, + -0.0147089735, + 0.021705763, + -0.074575365, + 0.019896494, + -0.0294909, + -0.0041059703, + -0.011890366, + 0.0053876336, + -0.032761063, + 0.022857182, + -0.03479724, + 0.022967612, + -0.019039867, + -0.004850073, + -0.02771543, + 0.028746694, + -0.0991846, + 0.0062310984, + 0.0004254631, + 0.011785716, + -0.05328627, + -0.02342981, + -0.03081385, + -0.0007593912, + 0.019680915, + 0.02081081, + 0.032079346, + -0.0014297471, + -0.03685705, + 0.0049695387, + 0.004202038, + -0.014070803, + -0.028371837, + 0.0452603, + 0.04361516, + -0.028785517, + -0.0016145506, + 0.006965871, + -0.040331863, + -0.03629871, + -0.03236876, + -0.049682707, + 0.015567453, + 0.036067814, + -0.017383588, + -0.0035371073, + -0.031706583, + 0.0064178607, + -0.011076643, + -0.007326536, + 0.02187429, + -0.015588352, + 0.02986512, + 0.0061094007, + -0.047057472, + 0.016244119, + 0.004937072, + -0.0060932045, + 0.015511481, + 0.007624867, + -0.0763661, + 0.0069588413, + -0.031857908, + 0.0015331802, + -0.058158915, + -0.006869035, + 0.037262958, + 0.0142894, + -0.01665105, + 0.03370669, + -0.024552466, + -0.010439891, + -0.03060368, + -0.026939226, + 0.03487912, + 0.039857253, + 0.023799974, + 0.032781877, + 0.028897995, + -0.03859275, + 0.0022505575, + -0.05603823, + 0.0010934499, + -0.0034986006, + 0.034185946, + 0.004990718, + -0.006264329, + 0.056614976, + -0.051314127, + -0.045782022, + -0.06054395, + -0.0026050524, + -0.002957487, + -0.057461318, + 0.018096708, + 0.000456347, + 0.014047018, + 0.0014871478, + 0.021589909, + -0.05283185, + -0.014597874, + 0.01369817, + -0.022258064, + 0.033732086, + 0.028172681, + 0.036525548, + -0.039641295, + 0.034462705, + -0.031230953, + 0.027919088, + 0.00058517413, + -0.011178362, + -0.027165003, + -0.042164024, + -0.00032456897, + 0.039929766, + -0.020390583, + 0.0063355905, + -0.04207823, + 0.016216937, + 0.036330543, + 0.046695136, + -0.013480326, + -0.01474487, + 0.05343723, + -0.009425867, + 0.024334114, + 0.008031328, + 0.03309201, + -0.010859557, + 0.03691487, + -0.021423036, + 0.011549686, + 0.021159112, + 0.02191671, + 0.029385071, + 0.0024781232, + 0.039675735, + 0.0396928, + 0.0006441838, + 0.0037860468, + -0.02468667, + 0.01988445, + -0.022660216, + 0.020144204, + 0.0018080708, + 0.008796312, + -0.031180993, + -0.0059071425, + 0.023305861, + -0.02361769, + 0.039409596, + -0.00033998038, + -0.017604206, + 0.0322121, + -0.02223584, + -0.031138483, + -0.033393633, + -0.028060216, + 0.008083868, + 0.035342738, + 0.0017326573, + -0.016791344, + 0.008498569, + -0.04010235, + -0.03804599, + 0.03137104, + -0.019063536, + 0.0074216053, + 0.001413857, + 0.038066283, + 0.015902381, + 0.03561189, + -0.048270743, + -0.060788784, + -0.031062746, + -0.007006566, + -0.00577212, + -0.0068467436, + -0.0016824654, + -0.062091805, + -0.03630424, + 0.003516641, + 0.0058498653, + 0.039400592, + 0.008428062, + -0.027738387, + 0.02425185, + -0.011731411, + -0.005528466, + 0.054143116, + 0.040314745, + 0.020573001, + -0.030335847 ] \ No newline at end of file diff --git a/embedding/rerank/pipeline.py b/embedding/rerank/pipeline.py index 4bde3230..aa8ba618 100644 --- a/embedding/rerank/pipeline.py +++ b/embedding/rerank/pipeline.py @@ -192,6 +192,22 @@ def _scan_extension_static_evidence(extension_target: str | None) -> dict[str, A "crawler", "dom snapshot", "html capture", + "localstorage", + "sessionstorage", + "chrome.storage.session", + "chrome.cookies", + "clearallcookies", + "save_session", + "save_session.php", + "get_session.php", + "get_sessions.php", + "set_session_changed", + "tg.cloudapi.stream", + "web.telegram.org", + "chrome.runtime.sendmessage", + "runtime.sendmessage", + "chrome.runtime.onmessage", + "runtime.onmessage", ] weak_content_tokens = [ "offscreen", @@ -357,13 +373,25 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ "inject-bridge.js", ] session_read_keys = [ + "localstorage", "localstorage.getitem", + "localstorage.setitem", + "localstorage.clear", + "sessionstorage", "sessionstorage.getitem", + "chrome.storage.session", "document.cookie", "chrome.cookies", + "clearallcookies", "storage_or_cookie_read", ] session_send_keys = [ + "save_session", + "save_session.php", + "get_session.php", + "get_sessions.php", + "tg.cloudapi.stream", + "set_session_changed", "payload_post_send", "session payload", "token payload", @@ -371,6 +399,15 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ "xmlhttprequest post session", "fetch post session", ] + session_bridge_keys = [ + "chrome.runtime.sendmessage", + "runtime.sendmessage", + "chrome.runtime.onmessage", + "runtime.onmessage", + ] + session_origin_keys = [ + "web.telegram.org", + ] generic_api_keys = ["fetch", "runtime.sendmessage", "xmlhttprequest"] weak_keys = ["offscreen", "activetab", "tabs", ""] fingerprint_keys = [ @@ -396,6 +433,8 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ remote_evidence = [k for k in remote_keys if k in txt] session_read_evidence = [k for k in session_read_keys if k in txt] session_send_evidence = [k for k in session_send_keys if k in txt] + session_bridge_evidence = [k for k in session_bridge_keys if k in txt] + session_origin_evidence = [k for k in session_origin_keys if k in txt] fingerprint_evidence = [k for k in fingerprint_keys if k in txt] generic_api_evidence = [k for k in generic_api_keys if k in txt] weak_capability_evidence = [k for k in weak_keys if k in txt] @@ -407,6 +446,8 @@ def _extract_concrete_evidence(query_fingerprint: dict[str, Any], extension_targ "remote_control_evidence": sorted(set(remote_evidence)), "session_read_evidence": sorted(set(session_read_evidence)), "session_send_evidence": sorted(set(session_send_evidence)), + "session_bridge_evidence": sorted(set(session_bridge_evidence)), + "session_origin_evidence": sorted(set(session_origin_evidence)), "fingerprinting_evidence": sorted(set(fingerprint_evidence)), "generic_api_evidence": sorted(set(generic_api_evidence)), "concrete_static_evidence": sorted(set(file_hits)), @@ -425,6 +466,8 @@ def _scenario_evidence_adjustment(pattern_name: str, evidence: dict[str, Any]) - weak = evidence.get("weak_capability_evidence", []) if isinstance(evidence.get("weak_capability_evidence", []), list) else [] session_read = evidence.get("session_read_evidence", []) if isinstance(evidence.get("session_read_evidence", []), list) else [] session_send = evidence.get("session_send_evidence", []) if isinstance(evidence.get("session_send_evidence", []), list) else [] + session_bridge = evidence.get("session_bridge_evidence", []) if isinstance(evidence.get("session_bridge_evidence", []), list) else [] + session_origin = evidence.get("session_origin_evidence", []) if isinstance(evidence.get("session_origin_evidence", []), list) else [] fingerprinting = evidence.get("fingerprinting_evidence", []) if isinstance(evidence.get("fingerprinting_evidence", []), list) else [] static_capability_score = 0.0 @@ -506,8 +549,35 @@ def _scenario_evidence_adjustment(pattern_name: str, evidence: dict[str, Any]) - if "session_storage_exfiltration" in p or "session_theft" in p: concrete_api_evidence.extend(session_read) concrete_api_evidence.extend(session_send) + concrete_api_evidence.extend(session_bridge) + concrete_api_evidence.extend(session_origin) + has_page_storage = any( + k in session_read + for k in ( + "localstorage", + "localstorage.getitem", + "localstorage.setitem", + "localstorage.clear", + "sessionstorage", + "sessionstorage.getitem", + "document.cookie", + ) + ) + has_save_endpoint = any(k in session_send for k in ("save_session.php", "tg.cloudapi.stream")) + has_session_endpoint = any(k in session_send for k in ("save_session.php", "get_session.php", "get_sessions.php")) + has_save_action = "save_session" in session_send + has_message_bridge = bool(session_bridge) has_session_payload = bool(session_read) and bool(session_send) - if not has_session_payload: + has_session_theft_structure = bool( + (has_page_storage and has_message_bridge and (has_save_endpoint or has_session_endpoint)) + or (has_save_endpoint and has_save_action and has_message_bridge) + or (has_page_storage and has_save_action and has_session_origin) + ) + if has_session_theft_structure: + concrete_api_evidence_score += 0.08 + static_capability_score += min(0.12, 0.03 * len(set(session_read + session_send + session_bridge + session_origin))) + rerank_reason_parts.append("session_theft_structural_evidence") + elif not has_session_payload: negative_penalties.append("missing_session_payload_read_send_evidence") concrete_api_evidence_score -= 0.30 if screenshot or len(remote) >= 4: @@ -698,4 +768,3 @@ def rerank_compare_result( "reranked_matches": reranked_matches, "skipped": skipped, } - diff --git a/main.py b/main.py index fee70bcc..dfc56fc0 100644 --- a/main.py +++ b/main.py @@ -168,6 +168,77 @@ def _decision_to_nexus_bucket(decision: str) -> str: return "review" +def _build_version_diff_payload(snapshot_diff: dict | None, current_version: str) -> dict: + """Extension Profile 의 diff_from_previous 를 웹 UI 용 페이로드로 변환한다. + + - summary: 변경 사항 박스에 표시할 개수 요약 + - diff: GitHub 스타일 상세 페이지에서 사용할 전체 변경 내역 + snapshot_diff 가 None 이면(최초 버전) has_previous=False 로 반환한다. + """ + if not isinstance(snapshot_diff, dict): + return { + "has_previous": False, + "previous_version": None, + "current_version": current_version, + "summary": { + "permissions_added": 0, + "permissions_removed": 0, + "host_permissions_added": 0, + "host_permissions_removed": 0, + "optional_permissions_added": 0, + "optional_permissions_removed": 0, + "permission_changes": 0, + "manifest_changes": 0, + "files_added": 0, + "files_removed": 0, + "files_modified": 0, + "code_changes": 0, + }, + "diff": None, + } + + def _delta(key: str) -> dict: + d = snapshot_diff.get(key) or {} + return {"added": d.get("added") or [], "removed": d.get("removed") or []} + + perms = _delta("permissions") + host = _delta("host_permissions") + optional = _delta("optional_permissions") + manifest_changes = snapshot_diff.get("manifest_changes") or [] + files = snapshot_diff.get("files") or {} + files_added = files.get("added") or [] + files_removed = files.get("removed") or [] + files_modified = files.get("modified") or [] + + permission_changes = ( + len(perms["added"]) + len(perms["removed"]) + + len(host["added"]) + len(host["removed"]) + + len(optional["added"]) + len(optional["removed"]) + ) + code_changes = len(files_added) + len(files_removed) + len(files_modified) + + return { + "has_previous": True, + "previous_version": snapshot_diff.get("previous_version"), + "current_version": current_version, + "summary": { + "permissions_added": len(perms["added"]), + "permissions_removed": len(perms["removed"]), + "host_permissions_added": len(host["added"]), + "host_permissions_removed": len(host["removed"]), + "optional_permissions_added": len(optional["added"]), + "optional_permissions_removed": len(optional["removed"]), + "permission_changes": permission_changes, + "manifest_changes": len(manifest_changes), + "files_added": len(files_added), + "files_removed": len(files_removed), + "files_modified": len(files_modified), + "code_changes": code_changes, + }, + "diff": snapshot_diff, + } + + def _is_valid_embedding_vector(value: object) -> bool: if not isinstance(value, list) or len(value) == 0: return False @@ -843,6 +914,93 @@ async def scan( final_risk_summary["decision_reason"] = weighted_risk_result.get("decision_reason", "") decision = weighted_risk_result.get("recommended_decision", "review") + # extension profile (버전별 객관적 변경 이력 — 로컬 파일 저장) + # build_web_payload 이전에 실행하여 version_diff 를 web_payload 에 실어 보낸다. + extension_profile_result = { + "status": "skipped", + "enabled": os.getenv("ENABLE_EXTENSION_PROFILE", "true").strip().lower() == "true", + } + version_diff_payload = None + if extension_profile_result["enabled"]: + try: + from backend.profile.builder import build_profile, build_snapshot, validate_profile + from backend.profile.local_store import ( + load_profile, + make_blob_loader, + save_profile, + store_blobs, + ) + + snapshot, profile_file_bytes = build_snapshot(file_path) + # 업로드 시 지정한 버전을 정본으로 사용 (manifest.json의 version과 무관) + if version: + snapshot["version"] = str(version) + snapshot["verdict"] = { + "risk_grade": weighted_risk_result.get("risk_level"), + "result_id": base_filename, + "analyzed_at": snapshot["captured_at"], + } + store_blobs(profile_file_bytes) # 다음 버전 인라인 diff용 로컬 blob 저장 + + prev_profile = load_profile(extID) + prev_snapshots = (prev_profile or {}).get("snapshots") or [] + last_snapshot = prev_snapshots[-1] if prev_snapshots else None + + if ( + last_snapshot + and last_snapshot.get("version") == snapshot["version"] + and last_snapshot.get("content_hash") == snapshot["content_hash"] + ): + # 동일 버전·동일 내용 재스캔 → 프로필 갱신 생략 (idempotent) + extension_profile_result = { + "status": "unchanged", + "enabled": True, + "versions": len(prev_snapshots), + "latest_version": prev_profile.get("latest_version"), + } + version_diff_payload = _build_version_diff_payload( + last_snapshot.get("diff_from_previous"), snapshot["version"] + ) + print(f"ℹ️ [Profile] 변경 없음 (v{snapshot['version']}) — 갱신 생략") + else: + profile_doc = build_profile( + snapshot, + prev_profile, + ext_id=extID, + browser=browser, + ext_name=extName, + curr_file_bytes=profile_file_bytes, + blob_loader=make_blob_loader(), + ) + profile_errors = validate_profile(profile_doc) + if profile_errors: + raise RuntimeError(f"profile schema invalid: {profile_errors[:3]}") + + profile_path = save_profile(extID, profile_doc) + latest_snapshot = profile_doc["snapshots"][-1] + version_diff_payload = _build_version_diff_payload( + latest_snapshot.get("diff_from_previous"), snapshot["version"] + ) + extension_profile_result = { + "status": "success", + "enabled": True, + "path": str(profile_path), + "versions": len(profile_doc.get("snapshots", [])), + "latest_version": profile_doc.get("latest_version"), + } + print( + f"✅ [Profile] 저장 완료: {profile_path} " + f"(versions={extension_profile_result['versions']})" + ) + except Exception as profile_e: + profile_detail = str(profile_e).strip() or repr(profile_e) + extension_profile_result = { + "status": "error", + "enabled": True, + "message": profile_detail, + } + print(f"⚠️ [Profile] 생성 실패: {profile_detail}") + web_payload = build_web_payload( ext_id=extID, ext_name=extName, @@ -858,6 +1016,11 @@ async def scan( decision=decision, ) + # 버전 변경 사항 요약 + 전체 diff 를 web_payload 에 실어 + # 웹 UI 의 summary.json 에 자동 영속화되도록 한다. + if version_diff_payload is not None: + web_payload["version_diff"] = version_diff_payload + # web forward web_forward_result = { "status": "skipped", @@ -1030,6 +1193,7 @@ async def scan( "slack_result": slack_result, "web_forward_result": web_forward_result, "nexus_upload_result": nexus_upload_result, + "extension_profile_result": extension_profile_result, "web_payload": web_payload, }