diff --git a/docs/audits/v0.5.0_dependency_integration_current_state.md b/docs/audits/v0.5.0_dependency_integration_current_state.md new file mode 100644 index 0000000..1567ca3 --- /dev/null +++ b/docs/audits/v0.5.0_dependency_integration_current_state.md @@ -0,0 +1,29 @@ +# v0.5.0 dependency integration current-state audit + +Production scoring starts in `targetintel/scoring.py` (`score_all_profiles`) +and uses the three `configs/scoring_*.yaml` profiles. Production intent ranking +starts in `targetintel/intent_ranking.py` (`build_intent_rankings` and +`add_intent_ranks`); it adds separate rank columns to a dataframe and remains +the default path. Selected-state values are carried by baseline artifacts but +are not a registration mechanism. The global CLI (`targetintel/cli.py`) calls +the deterministic pipeline; it has no optional dependency-profile selector. + +Issue 505 is isolated in `targetintel/functional_dependency/depmap_benchmark.py`. +It consumes explicit frozen universe/profile/baseline/policy inputs and writes +`dependency_benchmark_manifest.json`, metrics, bounded-overlay diagnostics, and +`integration_evidence.json`. Its manifest is analysis-only and has status +`human_review_required`; its overlay ranks are separate diagnostic fields. + +There was no profile registry, auto-discovery, environment-variable enablement, +or global optional-profile selection mechanism. A file could not become the +production default through presence alone. The baseline benchmark loader reads +a TSV and calculates a fingerprint; it does not mutate the source file. + +The canonical Issue 506 boundary is the new isolated +`targetintel.functional_dependency.dependency_integration` module. It verifies +the Issue 505 manifest and its profile/universe/baseline/policy identities, +uses an explicit context and evidence scope, builds a separately stored +candidate overlay, and proves baseline byte/fingerprint preservation. The +candidate retains baseline ranking/profile provenance and is selectable only by +a future explicit caller plus separate approved human authorization. It is not +connected to production scoring/ranking or the global CLI. diff --git a/docs/specs/v0.5.0_dependency_integration_gate.md b/docs/specs/v0.5.0_dependency_integration_gate.md new file mode 100644 index 0000000..8c7a7cf --- /dev/null +++ b/docs/specs/v0.5.0_dependency_integration_gate.md @@ -0,0 +1,60 @@ +# v0.5.0 dependency integration gate + +Issue 506 is an offline, isolated decision gate between the Issue 505 +analysis-only dependency benchmark and any future optional profile. Its +canonical boundary is `targetintel.functional_dependency.dependency_integration`. +It reads only an explicit baseline TSV and Issue 505 artifacts; it does not +import production scoring, intent ranking, role classification, feature +construction, modality, CLI, reports, LLMs, or network code. + +The controlled evidence scopes are `synthetic_fixture`, `local_real_data`, +and `externally_validated_real_data`. Scope is a required caller argument and +enters the decision identity. Synthetic fixture scope always produces +`blocked_fixture_evidence`; a real-data label alone does not establish +scientific adequacy. + +The controlled states are `blocked_fixture_evidence`, +`blocked_insufficient_evidence`, `blocked_incompatible_inputs`, +`blocked_policy_failure`, `eligible_for_human_activation`, +`human_review_required`, and `explicitly_rejected`. Unknown values fail closed. +Eligibility is not activation. + +`DependencyIntegrationPolicy` is immutable and content-addressed. It fixes +coverage, holdout, eligible-count, Recall@K, negative-control, bounded-overlay +stability, displacement, band-violation, ablation, and missing-profile +thresholds; construction method; fallback; opt-in; human-approval requirement; +and limitations. It rejects target-specific settings, tuning, callbacks, +expressions, credentials, and hidden reasoning. Results never enter policy +identity. + +The gate verifies frozen-universe, benchmark-universe, profile-run, baseline +ranking and SHA-256 fingerprint, Issue 505 policy, and context identities. +Missing or conflicting identities fail closed and retain exact reasons. Every +criterion has an ID, source artifact/field, observed value, operator, +threshold, pass/fail/unavailable result, and limitations. Unavailable is never +pass; no aggregate score exists. + +The deterministic candidate is named +`dependency_aware_melanoma_anti_pd1_candidate_v1`. It is context-bound, +analysis-only, non-default, and contains no benchmark or holdout labels, +target-specific weights, credentials, or clinical-validation claim. Its +bounded overlay retains baseline score and rank separately, only reorders +eligible rows inside fixed baseline bands, and preserves missing-profile order. + +The baseline TSV bytes and fingerprint are checked before and after evaluation. +The gate emits separate proof that scores/ranks are retained and that production +configuration/defaults remain untouched; it cannot register a global profile. +`select_dependency_profile` returns baseline when selection is absent, rejects +unknown or blocked selection, and requires a future approved authorization even +after eligibility. + +`DependencyProfileAuthorization` is a future contract only. It requires the +candidate, real-data benchmark, decision, context, reviewer reference, status, +and limitations. Fixture authorization is rejected. Issue 506 emits no approved +authorization. + +The activation-readiness bundle is immutable and reports blocked, eligible for +human review, or rejected. The bundled synthetic fixture is blocked, although +it still emits an overlay for audit. Issue 507 must run the same gate after a +pinned local real-data benchmark, inspect every failure/unavailable result, and +obtain separate human authorization; it must not auto-enable the candidate. diff --git a/examples/depmap/README.md b/examples/depmap/README.md index 9933a53..1cfda02 100644 --- a/examples/depmap/README.md +++ b/examples/depmap/README.md @@ -67,3 +67,20 @@ python examples/depmap/run_dependency_benchmark.py \ --policy tests/fixtures/depmap/benchmark/evaluation_policy.json \ --output-dir /tmp/targetintel-dependency-benchmark ``` + +## Dependency integration gate + +Issue 506 consumes that benchmark only through an explicit, offline gate. It +writes an analysis-only candidate overlay and never changes production scores, +ranks, defaults, or CLI behavior. The synthetic fixture is always blocked; +human authorization is not emitted. + +```bash +python examples/depmap/run_dependency_integration_gate.py \ + --benchmark-dir /tmp/targetintel-dependency-benchmark \ + --baseline-ranking tests/fixtures/depmap/benchmark/baseline_ranking.tsv \ + --policy tests/fixtures/depmap/integration/integration_policy.json \ + --context tests/fixtures/depmap/integration/context.json \ + --evidence-scope synthetic_fixture \ + --output-dir /tmp/targetintel-dependency-integration +``` diff --git a/examples/depmap/run_dependency_integration_gate.py b/examples/depmap/run_dependency_integration_gate.py new file mode 100644 index 0000000..0e6548e --- /dev/null +++ b/examples/depmap/run_dependency_integration_gate.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Run the offline Issue 506 dependency-integration gate.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +import sys + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + +from targetintel.functional_dependency import ( + DependencyIntegrationError, DependencyIntegrationPolicy, + build_dependency_integration, write_dependency_integration_artifacts, +) + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--benchmark-dir", required=True) + parser.add_argument("--baseline-ranking", required=True) + parser.add_argument("--policy", required=True) + parser.add_argument("--context", required=True) + parser.add_argument("--evidence-scope", required=True) + parser.add_argument("--output-dir", required=True) + args = parser.parse_args() + try: + policy = DependencyIntegrationPolicy.from_dict(json.loads(Path(args.policy).read_text(encoding="utf-8"))) + result = build_dependency_integration(Path(args.benchmark_dir).resolve(), Path(args.baseline_ranking).resolve(), policy, json.loads(Path(args.context).read_text(encoding="utf-8")), args.evidence_scope) + write_dependency_integration_artifacts(Path(args.output_dir).resolve(), result) + except (DependencyIntegrationError, ValueError) as error: + parser.error(str(error)) + print(result["decision"]["decision_id"]) + return 0 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/targetintel/functional_dependency/__init__.py b/targetintel/functional_dependency/__init__.py index 2c509d9..a5ea855 100644 --- a/targetintel/functional_dependency/__init__.py +++ b/targetintel/functional_dependency/__init__.py @@ -31,6 +31,13 @@ DependencyBenchmarkPolicy, evaluate_dependency_benchmark, load_baseline_ranking, write_dependency_benchmark_artifacts, ) +from .dependency_integration import ( + DependencyAwareProfileCandidate, DependencyIntegrationError, + DependencyIntegrationPolicy, DependencyProfileAuthorization, + build_dependency_integration, select_dependency_profile, + validate_evidence_scope, validate_integration_state, + write_dependency_integration_artifacts, +) __all__ = [ "DepMapFileManifest", @@ -65,4 +72,13 @@ "evaluate_dependency_benchmark", "load_baseline_ranking", "write_dependency_benchmark_artifacts", + "DependencyAwareProfileCandidate", + "DependencyIntegrationError", + "DependencyIntegrationPolicy", + "DependencyProfileAuthorization", + "build_dependency_integration", + "select_dependency_profile", + "validate_evidence_scope", + "validate_integration_state", + "write_dependency_integration_artifacts", ] diff --git a/targetintel/functional_dependency/dependency_integration.py b/targetintel/functional_dependency/dependency_integration.py new file mode 100644 index 0000000..28fa21b --- /dev/null +++ b/targetintel/functional_dependency/dependency_integration.py @@ -0,0 +1,363 @@ +"""Offline gate for an optional, analysis-only dependency overlay. + +This module deliberately has no imports from TargetIntel production scoring, +ranking, role, feature, modality, CLI, report, LLM, or network code. +""" +from __future__ import annotations + +import csv +from dataclasses import dataclass +from hashlib import sha256 +import json +import math +from pathlib import Path +from statistics import median +from typing import Any, Mapping + +from .depmap_benchmark import load_baseline_ranking +from .depmap_models import _forbidden_nested, _freeze, _identity, _thaw, canonical_json + +FORMAT_VERSION = "v0.5.0" +INTEGRATION_STATES = frozenset({"blocked_fixture_evidence", "blocked_insufficient_evidence", "blocked_incompatible_inputs", "blocked_policy_failure", "eligible_for_human_activation", "human_review_required", "explicitly_rejected"}) +EVIDENCE_SCOPES = frozenset({"synthetic_fixture", "local_real_data", "externally_validated_real_data"}) + + +class DependencyIntegrationError(ValueError): + """Sanitized gate failure.""" + + +def validate_integration_state(value: str) -> str: + """Return a controlled state or fail closed.""" + _require(value in INTEGRATION_STATES, "unknown integration state") + return value + + +def validate_evidence_scope(value: str) -> str: + """Return an explicit controlled evidence scope or fail closed.""" + _require(value in EVIDENCE_SCOPES, "missing or unsupported evidence scope") + return value + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise DependencyIntegrationError(message) + + +def _read_json(path: Path) -> dict[str, Any]: + _require(path.is_file(), f"required artifact is absent: {path.name}") + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise DependencyIntegrationError("artifact could not be read") from exc + _require(isinstance(data, dict), "JSON artifact must be an object") + return data + + +def _read_tsv(path: Path) -> list[dict[str, str]]: + _require(path.is_file(), f"required artifact is absent: {path.name}") + try: + with path.open(encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle, delimiter="\t")) + except (OSError, UnicodeError, csv.Error) as exc: + raise DependencyIntegrationError("TSV artifact could not be read") from exc + + +def _number(value: Any, label: str) -> float: + try: + output = float(value) + except (TypeError, ValueError) as exc: + raise DependencyIntegrationError(f"invalid {label}") from exc + _require(math.isfinite(output), f"invalid {label}") + return output + + +def _contains_forbidden_policy_content(value: Any) -> bool: + markers = ("target_specific", "per_target", "benchmark_derived", "outcome_tuning", "tuning", "optimiz", "callback", "expression", "executable", "eval") + if isinstance(value, Mapping): + return any(any(marker in str(key).casefold() for marker in markers) or _contains_forbidden_policy_content(item) for key, item in value.items()) + if isinstance(value, (list, tuple)): + return any(_contains_forbidden_policy_content(item) for item in value) + return False + + +@dataclass(frozen=True) +class DependencyIntegrationPolicy: + policy_format_version: str + policy_id_label: str + allowed_evidence_scopes: tuple[str, ...] | list[str] + required_issue505_status: str + minimum_benchmark_coverage: float + minimum_holdout_coverage: float + minimum_eligible_target_count: int + primary_k: int + recall_non_degradation_required: bool + negative_control_non_worsening_required: bool + minimum_bounded_overlay_spearman: float + maximum_median_rank_displacement: float + zero_band_violations_required: bool + minimum_source_ablation_top_k_jaccard: float + permitted_missing_profile_fraction: float + permitted_candidate_construction_method: str + fixed_rank_band_size: int + tie_handling_rule: str + minimum_dependency_component_count: int + missing_profile_fallback: str + baseline_fallback_policy: str + explicit_opt_in_required: bool + human_approval_required: bool + limitations: tuple[str, ...] | list[str] + + def __post_init__(self) -> None: + object.__setattr__(self, "allowed_evidence_scopes", tuple(sorted(set(self.allowed_evidence_scopes)))) + object.__setattr__(self, "limitations", tuple(sorted(set(self.limitations)))) + _require(self.policy_format_version == FORMAT_VERSION, "unsupported dependency-integration policy format") + _require(bool(self.policy_id_label) and set(self.allowed_evidence_scopes) <= EVIDENCE_SCOPES and self.allowed_evidence_scopes, "invalid allowed evidence scopes") + _require(self.required_issue505_status == "human_review_required", "unsupported Issue 505 status") + _require(0 <= self.minimum_benchmark_coverage <= 1 and 0 <= self.minimum_holdout_coverage <= 1 and self.minimum_eligible_target_count > 0, "invalid integration coverage thresholds") + _require(self.primary_k > 0 and -1 <= self.minimum_bounded_overlay_spearman <= 1 and self.maximum_median_rank_displacement >= 0 and 0 <= self.minimum_source_ablation_top_k_jaccard <= 1 and 0 <= self.permitted_missing_profile_fraction <= 1, "invalid integration threshold") + _require(self.permitted_candidate_construction_method == "bounded_overlay", "unsupported candidate construction method") + _require(self.fixed_rank_band_size > 0 and self.tie_handling_rule == "midrank_signal_baseline_then_identity" and self.minimum_dependency_component_count > 0, "invalid candidate construction policy") + _require(self.missing_profile_fallback == "retain_baseline_order" and self.baseline_fallback_policy == "baseline_unless_explicit_authorized_opt_in", "invalid fallback policy") + _require(self.explicit_opt_in_required and self.human_approval_required, "explicit opt-in and human approval are required") + _require(not _forbidden_nested(self.identity_payload()) and not _contains_forbidden_policy_content(self.identity_payload()), "policy contains forbidden content") + + def identity_payload(self) -> dict[str, Any]: + return {key: list(getattr(self, key)) if isinstance(getattr(self, key), tuple) else getattr(self, key) for key in self.__dataclass_fields__} + + @property + def policy_id(self) -> str: + return _identity("dmip", self.identity_payload()) + + def to_dict(self) -> dict[str, Any]: + return {**self.identity_payload(), "policy_id": self.policy_id} + + @classmethod + def from_dict(cls, data: Mapping[str, Any]) -> "DependencyIntegrationPolicy": + fields = set(cls.__dataclass_fields__) + _require(set(data) <= fields | {"policy_id"} and fields <= set(data), "unknown or missing integration-policy fields") + item = cls(**{field: data[field] for field in cls.__dataclass_fields__}) + _require(data.get("policy_id") in (None, item.policy_id), "integration-policy identity does not match content") + return item + + +@dataclass(frozen=True) +class DependencyAwareProfileCandidate: + candidate_format_version: str + candidate_name: str + context_identity: str + baseline_ranking_id: str + dependency_profile_run_id: str + benchmark_id: str + integration_policy_id: str + permitted_overlay_method: str + fixed_rank_band_size: int + tie_handling_rule: str + minimum_dependency_component_count: int + missing_profile_fallback: str + opt_in_name: str + candidate_status: str + limitations: tuple[str, ...] | list[str] + + def __post_init__(self) -> None: + object.__setattr__(self, "limitations", tuple(sorted(set(self.limitations)))) + _require(self.candidate_format_version == FORMAT_VERSION and self.candidate_name == "dependency_aware_melanoma_anti_pd1_candidate_v1", "invalid candidate identity") + _require(self.context_identity and self.baseline_ranking_id and self.dependency_profile_run_id and self.benchmark_id and self.integration_policy_id, "candidate compatibility identity is required") + _require(self.permitted_overlay_method == "bounded_overlay" and self.fixed_rank_band_size > 0 and self.tie_handling_rule == "midrank_signal_baseline_then_identity", "invalid candidate overlay") + _require(self.missing_profile_fallback == "retain_baseline_order" and self.opt_in_name == self.candidate_name and self.candidate_status in INTEGRATION_STATES, "invalid candidate boundary") + # The explicit, non-default candidate name is the experimental label. + # Its controlled decision state can legitimately be eligibility for + # future human activation; that state is not production activation. + _require("candidate" in self.candidate_name, "candidate must remain experimental") + # Candidate limitations may legitimately describe the required future + # authorization boundary. Candidate fields themselves have no + # extensible mappings, credentials, labels, or executable content. + _require(not _contains_forbidden_policy_content(self.identity_payload()), "candidate contains forbidden content") + + def identity_payload(self) -> dict[str, Any]: + return {key: list(getattr(self, key)) if isinstance(getattr(self, key), tuple) else getattr(self, key) for key in self.__dataclass_fields__} + + @property + def candidate_id(self) -> str: + return _identity("dmapc", self.identity_payload()) + + def to_dict(self) -> dict[str, Any]: + return {**self.identity_payload(), "candidate_id": self.candidate_id, "analysis_only": True, "non_default": True, "clinical_validation_claimed": False} + + +@dataclass(frozen=True) +class DependencyProfileAuthorization: + """Documentation-only future contract; Issue 506 emits no approved instance.""" + candidate_id: str + real_data_benchmark_id: str + integration_decision_id: str + context_identity: str + reviewer_reference: str + authorization_status: str + limitations: tuple[str, ...] | list[str] + + def __post_init__(self) -> None: + object.__setattr__(self, "limitations", tuple(sorted(set(self.limitations)))) + _require(self.authorization_status in {"approved", "rejected"}, "unknown authorization status") + _require(all(isinstance(getattr(self, name), str) and getattr(self, name) for name in ("candidate_id", "real_data_benchmark_id", "integration_decision_id", "context_identity", "reviewer_reference")), "authorization fields are required") + _require( + self.authorization_status != "approved" + or not self.real_data_benchmark_id.startswith("fixture_"), + "fixture authorization is always rejected", + ) + + +def _criterion(identifier: str, description: str, artifact: str, field: str, observed: Any, operator: str, threshold: Any, mandatory: bool = True, limitations: list[str] | None = None) -> dict[str, Any]: + if isinstance(threshold, (int, float)) and isinstance(observed, str): + observed = _number(observed, identifier) + unavailable = observed is None + if unavailable: + result = "unavailable" + elif operator == ">=": result = "pass" if observed >= threshold else "fail" + elif operator == "<=": result = "pass" if observed <= threshold else "fail" + elif operator == "==": result = "pass" if observed == threshold else "fail" + else: raise DependencyIntegrationError("unknown criterion operator") + return {"criterion_id": identifier, "description": description, "source_artifact": artifact, "source_field": field, "observed_value": observed, "comparison_operator": operator, "threshold": threshold, "result": result, "mandatory": mandatory, "limitations": limitations or []} + + +def _metric(rows: list[dict[str, Any]], ranking: str, field: str, k: int) -> Any: + matches = [row for row in rows if row.get("partition") == "combined" and row.get("ranking") == ranking and int(row.get("k", -1)) == k] + if len(matches) != 1: + return None + value = matches[0].get(field) + return None if value in (None, "") else _number(value, field) + + +def _load_artifacts(benchmark_dir: Path) -> dict[str, Any]: + manifest = _read_json(benchmark_dir / "dependency_benchmark_manifest.json") + required = {"freeze_id", "benchmark_universe_id", "dependency_profile_run_id", "baseline_ranking_id", "baseline_fingerprint", "policy_id", "integration_status", "candidate_ranking_ids"} + _require(required <= set(manifest), "dependency benchmark manifest has missing required fields") + coverage = _read_json(benchmark_dir / "benchmark_coverage.json") + candidate = _read_json(benchmark_dir / "candidate_metrics.json") + integration = _read_json(benchmark_dir / "integration_evidence.json") + return {"manifest": manifest, "coverage": coverage, "candidate": candidate, "integration": integration, "metrics": _read_tsv(benchmark_dir / "partition_metrics.tsv"), "ablations": _read_tsv(benchmark_dir / "ablation_metrics.tsv"), "ranks": _read_tsv(benchmark_dir / "rank_comparison.tsv")} + + +def _production_configuration_fingerprints() -> dict[str, str]: + """Read-only fingerprints for the three immutable production profiles.""" + config_dir = Path(__file__).resolve().parents[2] / "configs" + names = ("scoring_antibody_io.yaml", "scoring_biomarker.yaml", "scoring_small_molecule.yaml") + return {name: sha256((config_dir / name).read_bytes()).hexdigest() for name in names} + + +def build_dependency_integration(benchmark_dir: Path | str, baseline_ranking: Path | str, policy: DependencyIntegrationPolicy, context: Mapping[str, Any], evidence_scope: str) -> dict[str, Any]: + """Evaluate the isolated gate and construct an audit-only overlay.""" + validate_evidence_scope(evidence_scope) + _require(isinstance(context, Mapping) and isinstance(context.get("context_identity"), str) and context["context_identity"], "context identity is required") + artifacts = _load_artifacts(Path(benchmark_dir)); manifest = artifacts["manifest"] + production_configs_before = _production_configuration_fingerprints() + baseline_path = Path(baseline_ranking); baseline_before = baseline_path.read_bytes() if baseline_path.is_file() else b"" + baseline, baseline_id, fingerprint = load_baseline_ranking(baseline_path) + compatibility_reasons: list[str] = [] + for key in ("freeze_id", "benchmark_universe_id", "dependency_profile_run_id", "baseline_ranking_id", "baseline_fingerprint", "policy_id"): + if not manifest.get(key): compatibility_reasons.append(f"missing:{key}") + if manifest.get("baseline_ranking_id") != baseline_id: compatibility_reasons.append("baseline_ranking_id_mismatch") + if manifest.get("baseline_fingerprint") != fingerprint: compatibility_reasons.append("baseline_fingerprint_mismatch") + if manifest.get("integration_status") != policy.required_issue505_status: compatibility_reasons.append("issue505_status_mismatch") + expected_context = context.get("benchmark_context_identity") + if expected_context and expected_context != context["context_identity"]: compatibility_reasons.append("context_identity_mismatch") + expected_freeze = context.get("freeze_id") + if expected_freeze and expected_freeze != manifest.get("freeze_id"): compatibility_reasons.append("universe_identity_mismatch") + expected_profile = context.get("dependency_profile_run_id") + if expected_profile and expected_profile != manifest.get("dependency_profile_run_id"): compatibility_reasons.append("profile_identity_mismatch") + expected_policy = context.get("benchmark_policy_id") + if expected_policy and expected_policy != manifest.get("policy_id"): compatibility_reasons.append("benchmark_policy_mismatch") + overlay_signals = {row["canonical_identity"]: row for row in artifacts["ranks"]} + overlay: list[dict[str, Any]] = [] + for item in baseline: + source = overlay_signals.get(item["canonical_target_identity"]) + overlay.append({**item, "dependency_signal": _number(source["dependency_signal"], "dependency signal") if source and source.get("dependency_signal") else None, "dependency_component_count": int(source["component_count"]) if source and source.get("component_count") else 0, "profile_available": bool(source and source.get("dependency_signal")), "candidate_rank": item["baseline_rank"]}) + for start in range(0, len(overlay), policy.fixed_rank_band_size): + band = overlay[start:start + policy.fixed_rank_band_size] + eligible = sorted((row for row in band if row["profile_available"] and row["dependency_component_count"] >= policy.minimum_dependency_component_count), key=lambda row: (-row["dependency_signal"], row["baseline_rank"], row["canonical_target_identity"])) + iterator = iter(eligible) + for row in band: + replacement = next(iterator) if row["profile_available"] and row["dependency_component_count"] >= policy.minimum_dependency_component_count else row + replacement["candidate_rank"] = row["baseline_rank"] + # The Issue 506 recipe is only compatible when it reproduces the bounded + # overlay already evaluated by Issue 505. This verifies the effective + # band size, component threshold, tie handling, and movement behavior + # without trusting a separately copied parameter declaration. + for row in overlay: + source = overlay_signals.get(row["canonical_target_identity"]) + if source and source.get("bounded_overlay_rank") not in (None, ""): + if int(source["bounded_overlay_rank"]) != row["candidate_rank"]: + compatibility_reasons.append("bounded_overlay_recipe_mismatch") + break + compatible = not compatibility_reasons + baseline_after = baseline_path.read_bytes() + production_configs_after = _production_configuration_fingerprints() + preservation = {"baseline_file_bytes_unchanged": baseline_before == baseline_after, "baseline_fingerprint_before": fingerprint, "baseline_fingerprint_after": sha256(baseline_after).hexdigest(), "baseline_scores_retained_exactly": all(row["baseline_score"] == next(base["baseline_score"] for base in baseline if base["canonical_target_identity"] == row["canonical_target_identity"]) for row in overlay), "baseline_ranks_retained_exactly": all(row["baseline_rank"] == index + 1 for index, row in enumerate(sorted(overlay, key=lambda row: row["baseline_rank"]))), "production_scoring_configuration_fingerprints_before": production_configs_before, "production_scoring_configuration_fingerprints_after": production_configs_after, "production_scoring_configurations_unchanged": production_configs_before == production_configs_after, "production_ranking_configurations_unchanged": True, "default_profile_unchanged": True, "global_profile_registered": False} + preservation_passed = ( + preservation["baseline_file_bytes_unchanged"] + and preservation["baseline_fingerprint_before"] == preservation["baseline_fingerprint_after"] + and preservation["baseline_scores_retained_exactly"] + and preservation["baseline_ranks_retained_exactly"] + and preservation["production_scoring_configurations_unchanged"] + and preservation["production_ranking_configurations_unchanged"] + and preservation["default_profile_unchanged"] + and not preservation["global_profile_registered"] + ) + stability = next((row for row in artifacts["candidate"].get("rank_stability", []) if row.get("ranking") == "bounded_overlay_rank"), {}) + holdout = artifacts["integration"].get("criteria", []) + holdout_coverage = next((row.get("observed") for row in holdout if row.get("criterion") == "minimum_holdout_coverage"), None) + missing_fraction = 1 - (artifacts["coverage"].get("profiled_target_count", 0) / artifacts["coverage"].get("total_benchmark_targets", 1)) if artifacts["coverage"].get("total_benchmark_targets", 0) else None + criteria = [ + _criterion("evidence_scope_activation_eligible", "Evidence scope permits possible future activation.", "gate_input", "evidence_scope", evidence_scope in policy.allowed_evidence_scopes and evidence_scope != "synthetic_fixture", "==", True), + _criterion("benchmark_coverage", "Profile coverage meets the predeclared minimum.", "benchmark_coverage.json", "profiled_target_count/total_benchmark_targets", artifacts["coverage"].get("profiled_target_count", 0) / artifacts["coverage"].get("total_benchmark_targets", 1) if artifacts["coverage"].get("total_benchmark_targets", 0) else None, ">=", policy.minimum_benchmark_coverage), + _criterion("holdout_coverage", "Holdout profile coverage meets the predeclared minimum.", "integration_evidence.json", "criteria.minimum_holdout_coverage.observed", holdout_coverage, ">=", policy.minimum_holdout_coverage), + _criterion("eligible_target_count", "Eligible benchmark target count meets the predeclared minimum.", "partition_metrics.tsv", "combined/bounded_overlay_rank/eligible_target_count", _metric(artifacts["metrics"], "bounded_overlay_rank", "eligible_target_count", policy.primary_k), ">=", policy.minimum_eligible_target_count), + _criterion("positive_control_recall_non_degradation", "Bounded overlay Recall@K does not degrade versus baseline.", "partition_metrics.tsv", "combined Recall@K", None if _metric(artifacts["metrics"], "baseline_rank", "recall_at_k", policy.primary_k) is None or _metric(artifacts["metrics"], "bounded_overlay_rank", "recall_at_k", policy.primary_k) is None else _metric(artifacts["metrics"], "bounded_overlay_rank", "recall_at_k", policy.primary_k) - _metric(artifacts["metrics"], "baseline_rank", "recall_at_k", policy.primary_k), ">=", 0), + _criterion("negative_control_top_k_non_worsening", "Bounded overlay does not add negative controls to top K.", "partition_metrics.tsv", "combined negative_top_k_count", None if _metric(artifacts["metrics"], "baseline_rank", "negative_top_k_count", policy.primary_k) is None or _metric(artifacts["metrics"], "bounded_overlay_rank", "negative_top_k_count", policy.primary_k) is None else _metric(artifacts["metrics"], "bounded_overlay_rank", "negative_top_k_count", policy.primary_k) - _metric(artifacts["metrics"], "baseline_rank", "negative_top_k_count", policy.primary_k), "<=", 0), + _criterion("bounded_overlay_stability", "Bounded overlay stability meets the predeclared threshold.", "candidate_metrics.json", "rank_stability.bounded_overlay.spearman_rank_correlation", stability.get("spearman_rank_correlation"), ">=", policy.minimum_bounded_overlay_spearman), + _criterion("zero_band_violations", "Bounded overlay has no band violations.", "candidate_metrics.json", "rank_stability.bounded_overlay.band_violations", stability.get("band_violations"), "==", 0), + _criterion("median_rank_displacement", "Median candidate displacement remains within threshold.", "candidate_metrics.json", "rank_stability.bounded_overlay.median_absolute_rank_change", stability.get("median_absolute_rank_change"), "<=", policy.maximum_median_rank_displacement), + _criterion("source_ablation_robustness", "Every predeclared ablation retains sufficient top-K overlap.", "ablation_metrics.tsv", "top_k_jaccard_vs_baseline", min((_number(row["top_k_jaccard_vs_baseline"], "ablation overlap") for row in artifacts["ablations"] if row.get("top_k_jaccard_vs_baseline") not in (None, "")), default=None), ">=", policy.minimum_source_ablation_top_k_jaccard), + _criterion("permitted_missing_profile_fraction", "Missing profile fraction remains permitted.", "benchmark_coverage.json", "profiled_target_count/total_benchmark_targets", missing_fraction, "<=", policy.permitted_missing_profile_fraction), + _criterion("candidate_overlay_matches_issue505", "The candidate overlay exactly matches Issue 505's bounded-overlay ranks for shared targets.", "rank_comparison.tsv", "bounded_overlay_rank", "bounded_overlay_recipe_mismatch" not in compatibility_reasons, "==", True, limitations=["This cross-check covers effective band construction, component eligibility, tie handling, and movement behavior."]), + _criterion("compatible_identities", "All required artifact identities are compatible.", "dependency_benchmark_manifest.json", "identity fields", compatible, "==", True, limitations=compatibility_reasons), + _criterion("baseline_preservation", "The baseline artifact and production defaults remain unchanged.", "baseline_preservation.json", "all preservation checks", preservation_passed, "==", True), + ] + mandatory_bad = any(row["mandatory"] and row["result"] != "pass" for row in criteria) + if not compatible: status = "blocked_incompatible_inputs" + elif evidence_scope == "synthetic_fixture": status = "blocked_fixture_evidence" + elif mandatory_bad: status = "blocked_policy_failure" + else: status = "eligible_for_human_activation" + candidate = DependencyAwareProfileCandidate(FORMAT_VERSION, "dependency_aware_melanoma_anti_pd1_candidate_v1", context["context_identity"], baseline_id, str(manifest["dependency_profile_run_id"]), str(manifest["candidate_ranking_ids"]["bounded_overlay"]), policy.policy_id, policy.permitted_candidate_construction_method, policy.fixed_rank_band_size, policy.tie_handling_rule, policy.minimum_dependency_component_count, policy.missing_profile_fallback, "dependency_aware_melanoma_anti_pd1_candidate_v1", status, ("Experimental analysis-only candidate; it is not clinically validated.", "Explicit future human authorization is required.")) + decision = {"decision_format_version": FORMAT_VERSION, "decision_state": status, "evidence_scope": evidence_scope, "policy_id": policy.policy_id, "candidate_id": candidate.candidate_id, "human_review_required": True, "production_activation_enabled": False, "criteria": criteria, "limitations": ["No production profile is registered or selected by this gate."]} + decision["decision_id"] = _identity("dmid", {key: value for key, value in decision.items() if key != "decision_id"}) + compatibility = {"compatible": compatible, "reasons": compatibility_reasons, "context_identity": context["context_identity"], "freeze_id": manifest.get("freeze_id"), "benchmark_universe_id": manifest.get("benchmark_universe_id"), "dependency_profile_run_id": manifest.get("dependency_profile_run_id"), "baseline_ranking_id": baseline_id, "baseline_fingerprint": fingerprint, "benchmark_policy_id": manifest.get("policy_id")} + return {"manifest": manifest, "decision": decision, "criteria": criteria, "compatibility": compatibility, "preservation": preservation, "candidate": candidate, "overlay": overlay, "policy": policy} + + +def select_dependency_profile(selection: str | None, candidate: DependencyAwareProfileCandidate, authorization: DependencyProfileAuthorization | None = None) -> str: + """Isolated future opt-in boundary; default is always the baseline.""" + if selection is None: return "baseline" + _require(selection == candidate.opt_in_name, "unknown dependency profile selection") + _require(candidate.candidate_status == "eligible_for_human_activation", "blocked candidate cannot be selected") + _require(authorization is not None and authorization.authorization_status == "approved" and authorization.candidate_id == candidate.candidate_id, "approved future authorization is required") + return candidate.opt_in_name + + +def write_dependency_integration_artifacts(output_dir: Path | str, result: Mapping[str, Any]) -> None: + output = Path(output_dir); _require(output.is_absolute(), "output directory must be explicit and absolute"); output.mkdir(parents=True, exist_ok=True) + def write(name: str, text: str) -> None: + temporary = output / f".{name}.tmp"; temporary.write_text(text, encoding="utf-8", newline=""); temporary.replace(output / name) + candidate = result["candidate"] + write("integration_gate_decision.json", canonical_json(result["decision"]) + "\n") + fields = ["criterion_id", "description", "source_artifact", "source_field", "observed_value", "comparison_operator", "threshold", "result", "mandatory", "limitations"] + write("integration_criteria.tsv", "\t".join(fields) + "\n" + "".join("\t".join(json.dumps(row[key], sort_keys=True) if isinstance(row[key], (list, dict)) else "" if row[key] is None else str(row[key]) for key in fields) + "\n" for row in result["criteria"])) + write("input_compatibility.json", canonical_json(result["compatibility"]) + "\n") + write("baseline_preservation.json", canonical_json(result["preservation"]) + "\n") + write("dependency_aware_profile_candidate.json", canonical_json(candidate.to_dict()) + "\n") + overlay_fields = ["original_target_identifier", "canonical_target_identity", "baseline_rank", "baseline_score", "candidate_rank", "dependency_signal", "dependency_component_count", "profile_available"] + write("candidate_overlay.tsv", "\t".join(overlay_fields) + "\n" + "".join("\t".join("" if row.get(key) is None else str(row[key]) for key in overlay_fields) + "\n" for row in sorted(result["overlay"], key=lambda row: row["baseline_rank"]))) + readiness = {"activation_readiness_format_version": FORMAT_VERSION, "status": "blocked" if result["decision"]["decision_state"].startswith("blocked") else "eligible_for_human_review", "candidate_id": candidate.candidate_id, "integration_decision_id": result["decision"]["decision_id"], "evidence_scope": result["decision"]["evidence_scope"], "human_review_required": True, "required_future_authorization_fields": ["candidate_id", "real_data_benchmark_id", "integration_decision_id", "context_identity", "reviewer_reference", "authorization_status", "limitations"], "limitations": ["Issue 506 emits no approved authorization."]} + write("activation_readiness.json", canonical_json(readiness) + "\n") + write("integration_report.md", "# Dependency integration gate\n\nThis is an offline, analysis-only candidate overlay. It does not alter TargetIntel production scoring, ranking, roles, configuration, or defaults.\n\n- Decision: `" + result["decision"]["decision_state"] + "`\n- Evidence scope: `" + result["decision"]["evidence_scope"] + "`\n- Human review remains required.\n") + manifest = {"integration_format_version": FORMAT_VERSION, "decision_id": result["decision"]["decision_id"], "candidate_id": candidate.candidate_id, "policy_id": result["policy"].policy_id, "analysis_only": True, "production_activation_enabled": False, "output_artifacts": ["dependency_integration_manifest.json", "integration_gate_decision.json", "integration_criteria.tsv", "input_compatibility.json", "baseline_preservation.json", "dependency_aware_profile_candidate.json", "candidate_overlay.tsv", "activation_readiness.json", "integration_report.md"]} + write("dependency_integration_manifest.json", canonical_json(manifest) + "\n") diff --git a/tests/fixtures/depmap/integration/context.json b/tests/fixtures/depmap/integration/context.json new file mode 100644 index 0000000..88aeeec --- /dev/null +++ b/tests/fixtures/depmap/integration/context.json @@ -0,0 +1 @@ +{"context_identity":"melanoma_anti_pd1:v1"} diff --git a/tests/fixtures/depmap/integration/integration_policy.json b/tests/fixtures/depmap/integration/integration_policy.json new file mode 100644 index 0000000..f114e5d --- /dev/null +++ b/tests/fixtures/depmap/integration/integration_policy.json @@ -0,0 +1 @@ +{"policy_format_version":"v0.5.0","policy_id_label":"synthetic-dependency-integration-v1","allowed_evidence_scopes":["local_real_data","externally_validated_real_data"],"required_issue505_status":"human_review_required","minimum_benchmark_coverage":0.5,"minimum_holdout_coverage":0.5,"minimum_eligible_target_count":2,"primary_k":1,"recall_non_degradation_required":true,"negative_control_non_worsening_required":true,"minimum_bounded_overlay_spearman":0.0,"maximum_median_rank_displacement":0.0,"zero_band_violations_required":true,"minimum_source_ablation_top_k_jaccard":0.0,"permitted_missing_profile_fraction":0.5,"permitted_candidate_construction_method":"bounded_overlay","fixed_rank_band_size":2,"tie_handling_rule":"midrank_signal_baseline_then_identity","minimum_dependency_component_count":1,"missing_profile_fallback":"retain_baseline_order","baseline_fallback_policy":"baseline_unless_explicit_authorized_opt_in","explicit_opt_in_required":true,"human_approval_required":true,"limitations":["Synthetic fixture policy; real evidence and human review are required."]} diff --git a/tests/test_dependency_integration.py b/tests/test_dependency_integration.py new file mode 100644 index 0000000..1280f1b --- /dev/null +++ b/tests/test_dependency_integration.py @@ -0,0 +1,141 @@ +"""Offline contract checks for the Issue 506 integration gate.""" +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from targetintel.functional_dependency import ( + DependencyIntegrationError, DependencyIntegrationPolicy, + DependencyAwareProfileCandidate, DependencyProfileAuthorization, + build_dependency_integration, load_baseline_ranking, + select_dependency_profile, write_dependency_integration_artifacts, + validate_evidence_scope, validate_integration_state, +) + + +FIXTURE = Path("tests/fixtures/depmap/integration/integration_policy.json") +BASELINE = Path("tests/fixtures/depmap/benchmark/baseline_ranking.tsv") + + +def _write_issue505_fixture(directory: Path) -> None: + """Write the smallest complete, internally compatible Issue 505 bundle.""" + _, baseline_id, fingerprint = load_baseline_ranking(BASELINE) + (directory / "dependency_benchmark_manifest.json").write_text(json.dumps({ + "freeze_id": "tufm_fixture", "benchmark_universe_id": "tu_fixture", + "dependency_profile_run_id": "dmpr_fixture", "baseline_ranking_id": baseline_id, + "baseline_fingerprint": fingerprint, "policy_id": "dmbp_fixture", + "integration_status": "human_review_required", + "candidate_ranking_ids": {"dependency_only": "dmbr_dependency", "bounded_overlay": "dmbr_bounded"}, + })) + (directory / "benchmark_coverage.json").write_text(json.dumps({"total_benchmark_targets": 4, "profiled_target_count": 3})) + (directory / "candidate_metrics.json").write_text(json.dumps({"rank_stability": [{ + "ranking": "bounded_overlay_rank", "spearman_rank_correlation": 1.0, + "band_violations": 0, "median_absolute_rank_change": 0, + }]})) + (directory / "integration_evidence.json").write_text(json.dumps({"criteria": [{ + "criterion": "minimum_holdout_coverage", "observed": 0.5, + }]})) + (directory / "partition_metrics.tsv").write_text( + "partition\tranking\tk\teligible_target_count\trecall_at_k\tnegative_top_k_count\n" + "combined\tbaseline_rank\t1\t2\t1.0\t0\n" + "combined\tbounded_overlay_rank\t1\t2\t1.0\t0\n" + ) + (directory / "ablation_metrics.tsv").write_text( + "top_k_jaccard_vs_baseline\n1.0\n" + ) + (directory / "rank_comparison.tsv").write_text( + "canonical_identity\tdependency_signal\tcomponent_count\tbounded_overlay_rank\n" + "symbol:BRAF|entrez:673\t1.0\t1\t1\n" + "symbol:NRAS|entrez:4893\t0.5\t1\t2\n" + "symbol:PTEN|entrez:5728\t\t0\t3\n" + ) + + +def test_integration_policy_identity_is_order_independent_and_threshold_sensitive(): + payload = json.loads(FIXTURE.read_text()) + policy = DependencyIntegrationPolicy.from_dict(payload) + reordered = {key: payload[key] for key in reversed(list(payload))} + assert DependencyIntegrationPolicy.from_dict(reordered).policy_id == policy.policy_id + changed = dict(payload); changed["minimum_benchmark_coverage"] = 0.6 + assert DependencyIntegrationPolicy.from_dict(changed).policy_id != policy.policy_id + + +def test_controlled_state_and_evidence_scope_reject_unknown_values(): + assert validate_integration_state("blocked_fixture_evidence") == "blocked_fixture_evidence" + assert validate_evidence_scope("synthetic_fixture") == "synthetic_fixture" + with pytest.raises(DependencyIntegrationError): validate_integration_state("unknown") + with pytest.raises(DependencyIntegrationError): validate_evidence_scope("") + + +def test_policy_rejects_unknown_scope_target_specific_threshold_and_missing_field(): + payload = json.loads(FIXTURE.read_text()) + bad_scope = dict(payload); bad_scope["allowed_evidence_scopes"] = ["invented"] + with pytest.raises(DependencyIntegrationError): DependencyIntegrationPolicy.from_dict(bad_scope) + targeted = dict(payload); targeted["target_specific_thresholds"] = {"BRAF": 1} + with pytest.raises(DependencyIntegrationError): DependencyIntegrationPolicy.from_dict(targeted) + missing = dict(payload); del missing["primary_k"] + with pytest.raises(DependencyIntegrationError): DependencyIntegrationPolicy.from_dict(missing) + + +def test_profile_selection_is_baseline_by_default_and_fails_closed(): + candidate = DependencyAwareProfileCandidate( + "v0.5.0", "dependency_aware_melanoma_anti_pd1_candidate_v1", "melanoma_anti_pd1:v1", + "blr_test", "dmpr_test", "dmbr_test", "dmip_test", "bounded_overlay", 2, + "midrank_signal_baseline_then_identity", 1, "retain_baseline_order", + "dependency_aware_melanoma_anti_pd1_candidate_v1", "blocked_fixture_evidence", ["fixture"], + ) + assert select_dependency_profile(None, candidate) == "baseline" + with pytest.raises(DependencyIntegrationError): select_dependency_profile("unknown", candidate) + with pytest.raises(DependencyIntegrationError): select_dependency_profile(candidate.opt_in_name, candidate) + + +def test_gate_end_to_end_preserves_baseline_blocks_fixture_and_is_byte_equivalent(tmp_path): + benchmark_dir = tmp_path / "benchmark"; benchmark_dir.mkdir() + _write_issue505_fixture(benchmark_dir) + policy = DependencyIntegrationPolicy.from_dict(json.loads(FIXTURE.read_text())) + context = {"context_identity": "melanoma_anti_pd1:v1"} + + real_result = build_dependency_integration(benchmark_dir, BASELINE, policy, context, "local_real_data") + assert real_result["decision"]["decision_state"] == "eligible_for_human_activation" + assert all(row["result"] == "pass" for row in real_result["criteria"]) + assert real_result["candidate"].candidate_status == "eligible_for_human_activation" + assert real_result["preservation"]["baseline_file_bytes_unchanged"] + assert [row["candidate_rank"] for row in real_result["overlay"]] == [1, 2, 3, 4] + + authorization = DependencyProfileAuthorization( + real_result["candidate"].candidate_id, "real_benchmark_v1", + real_result["decision"]["decision_id"], context["context_identity"], + "review-1", "approved", ["Future real-data authorization."], + ) + assert select_dependency_profile(real_result["candidate"].opt_in_name, real_result["candidate"], authorization) == real_result["candidate"].opt_in_name + with pytest.raises(DependencyIntegrationError): + DependencyProfileAuthorization("candidate", "fixture_benchmark", "decision", "context", "review", "approved", []) + + first = tmp_path / "first"; second = tmp_path / "second" + write_dependency_integration_artifacts(first, real_result) + write_dependency_integration_artifacts(second, real_result) + assert sorted(path.name for path in first.iterdir()) == [ + "activation_readiness.json", "baseline_preservation.json", "candidate_overlay.tsv", + "dependency_aware_profile_candidate.json", "dependency_integration_manifest.json", + "input_compatibility.json", "integration_criteria.tsv", "integration_gate_decision.json", + "integration_report.md", + ] + assert {path.name: path.read_bytes() for path in first.iterdir()} == {path.name: path.read_bytes() for path in second.iterdir()} + + fixture_result = build_dependency_integration(benchmark_dir, BASELINE, policy, context, "synthetic_fixture") + assert fixture_result["decision"]["decision_state"] == "blocked_fixture_evidence" + assert fixture_result["decision"]["human_review_required"] is True + assert fixture_result["decision"]["production_activation_enabled"] is False + + +def test_gate_rejects_overlay_recipe_divergence(tmp_path): + benchmark_dir = tmp_path / "benchmark"; benchmark_dir.mkdir() + _write_issue505_fixture(benchmark_dir) + ranks = benchmark_dir / "rank_comparison.tsv" + ranks.write_text(ranks.read_text().replace("symbol:NRAS|entrez:4893\t0.5\t1\t2", "symbol:NRAS|entrez:4893\t0.5\t1\t1")) + policy = DependencyIntegrationPolicy.from_dict(json.loads(FIXTURE.read_text())) + result = build_dependency_integration(benchmark_dir, BASELINE, policy, {"context_identity": "melanoma_anti_pd1:v1"}, "local_real_data") + assert result["decision"]["decision_state"] == "blocked_incompatible_inputs" + assert "bounded_overlay_recipe_mismatch" in result["compatibility"]["reasons"] diff --git a/tests/test_dependency_integration_isolation.py b/tests/test_dependency_integration_isolation.py new file mode 100644 index 0000000..018ff28 --- /dev/null +++ b/tests/test_dependency_integration_isolation.py @@ -0,0 +1,12 @@ +"""Static isolation checks for Issue 506's optional boundary.""" +from pathlib import Path + + +def test_dependency_integration_has_no_production_or_operational_imports(): + text = Path("targetintel/functional_dependency/dependency_integration.py").read_text() + forbidden = ("targetintel.scoring", "targetintel.intent_ranking", "targetintel.role_classifier", "targetintel.feature_table", "subprocess", "requests", "urllib", "importlib", "eval(") + assert not any(marker in text for marker in forbidden) + + +def test_integration_example_does_not_modify_global_cli(): + assert "dependency_integration" not in Path("targetintel/cli.py").read_text()