From e4007bc12608913bb17e9d2a33b927e0789acb84 Mon Sep 17 00:00:00 2001 From: stevezkw <53121066+stevezkw1998@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:40:11 -0700 Subject: [PATCH 1/6] Invite external benchmark review --- README.de.md | 4 ++++ README.es.md | 4 ++++ README.fr.md | 4 ++++ README.ja.md | 4 ++++ README.ko.md | 4 ++++ README.md | 4 ++++ README.zh-CN.md | 4 ++++ README.zh-TW.md | 4 ++++ i18n/readme_state.json | 16 ++++++++-------- 9 files changed, 40 insertions(+), 8 deletions(-) diff --git a/README.de.md b/README.de.md index 98704c7..b7dbc9d 100644 --- a/README.de.md +++ b/README.de.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[Wir suchen aktiv externe Gutachterinnen und Gutachter sowie Autorinnen und Autoren von Prädiktoren.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> Plausibilitätsprüfungen von 15–30 Minuten, Reproduktionsversuche, Datensatzvorschläge und kritische Gegenprüfungen sind willkommen. + NeoRepro ist eine reproduzierbare, patientenbezogene Benchmark-Ressource für öffentliche MHC-I-Peptid–HLA-Neoantigen-Prädiktoren unter Berücksichtigung von Datenlecks. Sie bündelt festgeschriebene Prädiktor-Artefakte, Provenienz auf Datensatzebene, Prüfungen auf Überschneidungen mit Trainingsdaten, Vergleiche auf einer gemeinsamen Auswertungsmenge, patientenbezogene Unsicherheit, an die Abdeckung angepasste Zufallsreferenzen und maschinell erzeugte Ergebnisse. Der Beitrag ist eine Benchmark- und Forschungsressource, kein neuer Prädiktor und keine Behauptung eines universell besten Modells oder klinischen Nutzens. diff --git a/README.es.md b/README.es.md index 009f99f..9aff206 100644 --- a/README.es.md +++ b/README.es.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[Buscamos activamente revisores externos y autores de predictores.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> Agradecemos comprobaciones de coherencia de 15–30 minutos, intentos de reproducción, sugerencias de conjuntos de datos y críticas adversariales. + NeoRepro es un recurso de evaluación reproducible, sensible a la fuga de datos y a nivel de paciente para predictores públicos de neoantígenos péptido–HLA de MHC-I. Reúne artefactos de predictores fijados, procedencia por registro, auditorías de solapamiento con datos de entrenamiento, comparaciones sobre soporte común, incertidumbre a nivel de paciente, referencias aleatorias ajustadas al soporte y resultados generados automáticamente. Es una contribución de referencia y recursos, no un predictor nuevo ni una afirmación de que exista un modelo universalmente ganador o de utilidad clínica. diff --git a/README.fr.md b/README.fr.md index 3a8280e..1a187be 100644 --- a/README.fr.md +++ b/README.fr.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[Nous recherchons activement des évaluateurs externes et des auteurs de prédicteurs.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> Les vérifications de cohérence de 15–30 minutes, les tentatives de reproduction, les suggestions de jeux de données et les critiques contradictoires sont les bienvenues. + NeoRepro est une ressource d’évaluation reproductible, tenant compte des fuites de données et menée au niveau du patient, pour les prédicteurs publics de néoantigènes peptide–HLA de MHC-I. Elle rassemble des artefacts de prédicteurs épinglés, la provenance de chaque enregistrement, des audits de chevauchement avec les données d’entraînement, des comparaisons sur un support commun, l’incertitude au niveau du patient, des références aléatoires adaptées au support et des résultats générés automatiquement. Il s’agit d’une contribution de type ressource et benchmark, et non d’un nouveau prédicteur ni d’une affirmation concernant un modèle universellement gagnant ou une utilité clinique. diff --git a/README.ja.md b/README.ja.md index ee9012c..5ad42af 100644 --- a/README.ja.md +++ b/README.ja.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[外部レビュアーおよび予測器の作者を積極的に募集しています。](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 15–30 分の妥当性確認、再現の試行、データセットの提案、批判的な検証を歓迎します。 + NeoRepro は、公開されている MHC-I ペプチド–HLA ネオアンチゲン予測器を対象とした、リーケージを考慮した患者単位の再現可能なベンチマークリソースです。固定された予測器アーティファクト、レコード単位の来歴、学習データ重複監査、共通評価可能集合での比較、患者単位の不確実性、評価可能範囲を一致させたランダムベースライン、および機械生成結果をまとめています。 本プロジェクトはベンチマーク／リソースとしての貢献であり、新しい予測器ではありません。また、普遍的に最良のモデルや臨床的有用性を主張するものでもありません。 diff --git a/README.ko.md b/README.ko.md index c6e1f47..d7b4057 100644 --- a/README.ko.md +++ b/README.ko.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[외부 검토자와 예측기 개발자를 적극적으로 찾고 있습니다.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 15–30분의 타당성 점검, 재현 시도, 데이터셋 제안 및 비판적 검토를 환영합니다. + NeoRepro는 공개 MHC-I 펩타이드–HLA 신생항원 예측기를 위한 데이터 누출 인지형, 환자 수준, 재현 가능한 벤치마크 리소스입니다. 고정된 예측기 아티팩트, 레코드 수준 출처, 학습 데이터 중복 감사, 공통 평가 가능 집합 비교, 환자 수준 불확실성, 지원 범위를 맞춘 무작위 기준선 및 기계 생성 결과를 제공합니다. 이 프로젝트는 새로운 예측기가 아니라 벤치마크/리소스 기여이며, 보편적인 최우수 모델이나 임상적 유용성을 주장하지 않습니다. diff --git a/README.md b/README.md index 2d9cca1..b690f31 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[We are actively seeking external reviewers and predictor authors.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 15–30 min sanity checks, reproduction attempts, dataset suggestions and adversarial critiques are welcome. + NeoRepro is a leakage-aware, patient-level, reproducible benchmark resource for public MHC-I peptide–HLA neoantigen predictors. It packages pinned predictor artifacts, record-level provenance, training-overlap audits, common-support comparisons, patient-level uncertainty, support-matched random baselines and machine-generated results. This is a benchmark/resource contribution, not a new predictor and not a claim of a universal model winner or clinical utility. diff --git a/README.zh-CN.md b/README.zh-CN.md index f66438b..f7c98cc 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[我们正在积极寻找外部审阅者和预测器作者。](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 欢迎进行 15–30 分钟的合理性检查、复现尝试、数据集建议和对抗性批评。 + NeoRepro 是面向公开 MHC-I 肽–HLA 新抗原预测器的泄漏感知、患者级、可复现基准资源。它整合了固定版本的预测器工件、逐记录来源、训练数据重叠审计、共同支持集比较、患者级不确定性、支持集匹配的随机基线以及机器生成的结果。 本项目贡献的是基准与研究资源，而不是新的预测器，也不宣称存在普适的最佳模型或临床效用。 diff --git a/README.zh-TW.md b/README.zh-TW.md index 58563db..83867e7 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[我們正積極徵求外部審閱者與預測器作者。](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 歡迎進行 15–30 分鐘的合理性檢查、重現嘗試、資料集建議與對抗性批評。 + NeoRepro 是一套針對公開 MHC-I 胜肽–HLA 新抗原預測器、考量資料洩漏、病人層級且可重現的基準資源。它整合固定版本的預測器產物、逐筆資料來源、訓練資料重疊稽核、共同可評估集合比較、病人層級不確定性、依支援範圍匹配的隨機基準，以及由機器產生的結果。 本專案的貢獻是基準與研究資源，不是新的預測器，也不主張存在普遍適用的最佳模型或臨床效益。 diff --git a/i18n/readme_state.json b/i18n/readme_state.json index a2fd36e..7f891d3 100644 --- a/i18n/readme_state.json +++ b/i18n/readme_state.json @@ -6,7 +6,7 @@ "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "65418799adf89e3fce1e798f934e0d60ec21086d5db6023cf2721d1b841f0835", + "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" @@ -18,7 +18,7 @@ "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "65418799adf89e3fce1e798f934e0d60ec21086d5db6023cf2721d1b841f0835", + "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" @@ -30,7 +30,7 @@ "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "65418799adf89e3fce1e798f934e0d60ec21086d5db6023cf2721d1b841f0835", + "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" @@ -42,7 +42,7 @@ "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "65418799adf89e3fce1e798f934e0d60ec21086d5db6023cf2721d1b841f0835", + "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" @@ -54,7 +54,7 @@ "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "65418799adf89e3fce1e798f934e0d60ec21086d5db6023cf2721d1b841f0835", + "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" @@ -66,7 +66,7 @@ "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "65418799adf89e3fce1e798f934e0d60ec21086d5db6023cf2721d1b841f0835", + "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" @@ -78,7 +78,7 @@ "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "65418799adf89e3fce1e798f934e0d60ec21086d5db6023cf2721d1b841f0835", + "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" @@ -86,6 +86,6 @@ } }, "source": "README.md", - "source_sha256": "764f22611b5381d431e8191a5f19fb312f6023049bfbf942ea10e28a63243a12", + "source_sha256": "bed740fcc5071ac2386f37de6c67de7633a82ea72ade3cd40fcc211e888240d7", "version": 1 } From 6b54b65f96544218b5dfc58307a2d2901ae504ed Mon Sep 17 00:00:00 2001 From: stevezkw <53121066+stevezkw1998@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:50:31 -0700 Subject: [PATCH 2/6] Add deterministic repository reflexes --- .githooks/pre-commit | 38 ++++++++++++++++++++++++++++++++++++++ .githooks/pre-push | 9 +++++++++ 2 files changed, 47 insertions(+) create mode 100755 .githooks/pre-commit create mode 100755 .githooks/pre-push diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..25f0853 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu + +expected_name="stevezkw" +author_ident="$(git var GIT_AUTHOR_IDENT)" +committer_ident="$(git var GIT_COMMITTER_IDENT)" + +case "$author_ident" in + "$expected_name <"*) ;; + *) + echo "pre-commit: author must be '$expected_name'; got: $author_ident" >&2 + exit 1 + ;; +esac + +case "$committer_ident" in + "$expected_name <"*) ;; + *) + echo "pre-commit: committer must be '$expected_name'; got: $committer_ident" >&2 + exit 1 + ;; +esac + +git diff --cached --check + +changed_paths="$(git diff --cached --name-only --diff-filter=ACMR)" + +if printf '%s\n' "$changed_paths" | grep -Eq '^\.githooks/(pre-commit|pre-push)$'; then + sh -n .githooks/pre-commit .githooks/pre-push +fi + +if printf '%s\n' "$changed_paths" | grep -Eq '^(README(\.[^.]+)?\.md|i18n/(glossary\.md|readme_state\.json)|scripts/readme_i18n\.py|tests/test_readme_i18n\.py)$'; then + export PYTHONUTF8=1 + export LC_ALL=en_US.UTF-8 + export LANG=en_US.UTF-8 + python3 scripts/readme_i18n.py check + uv run pytest -q tests/test_readme_i18n.py +fi diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..0fe0e58 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +export PYTHONUTF8=1 +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 + +python3 scripts/readme_i18n.py check +uv run pytest -q From db00e769f8d6831221264e6a187309cfeadbd91a Mon Sep 17 00:00:00 2001 From: stevezkw <53121066+stevezkw1998@users.noreply.github.com> Date: Thu, 20 Aug 2026 17:29:03 -0700 Subject: [PATCH 3/6] Add standalone prediction audit command --- .githooks/pre-commit | 15 +- src/neorepro/audit.py | 475 ++++++++++++++++++++++++++++++++++++++++ src/neorepro/cli.py | 20 +- src/neorepro/metrics.py | 68 ++++++ tests/test_audit_cli.py | 111 ++++++++++ tests/test_metrics.py | 14 +- 6 files changed, 698 insertions(+), 5 deletions(-) create mode 100644 src/neorepro/audit.py create mode 100644 tests/test_audit_cli.py diff --git a/.githooks/pre-commit b/.githooks/pre-commit index 25f0853..51d32f2 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,6 +1,10 @@ #!/bin/sh set -eu +export PYTHONUTF8=1 +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 + expected_name="stevezkw" author_ident="$(git var GIT_AUTHOR_IDENT)" committer_ident="$(git var GIT_COMMITTER_IDENT)" @@ -30,9 +34,14 @@ if printf '%s\n' "$changed_paths" | grep -Eq '^\.githooks/(pre-commit|pre-push)$ fi if printf '%s\n' "$changed_paths" | grep -Eq '^(README(\.[^.]+)?\.md|i18n/(glossary\.md|readme_state\.json)|scripts/readme_i18n\.py|tests/test_readme_i18n\.py)$'; then - export PYTHONUTF8=1 - export LC_ALL=en_US.UTF-8 - export LANG=en_US.UTF-8 python3 scripts/readme_i18n.py check uv run pytest -q tests/test_readme_i18n.py fi + +if printf '%s\n' "$changed_paths" | grep -Eq '^(src|scripts|tests)/.*\.py$'; then + uv run ruff check src scripts tests +fi + +if printf '%s\n' "$changed_paths" | grep -Eq '^(src/neorepro/(audit|cli|metrics)\.py|tests/test_(audit_cli|metrics)\.py)$'; then + uv run pytest -q tests/test_audit_cli.py tests/test_metrics.py +fi diff --git a/src/neorepro/audit.py b/src/neorepro/audit.py new file mode 100644 index 0000000..30f70b9 --- /dev/null +++ b/src/neorepro/audit.py @@ -0,0 +1,475 @@ +"""Audit externally generated predictor scores against frozen NeoRepro benchmarks.""" + +from __future__ import annotations + +import csv +import math +import random +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from statistics import mean + +from neorepro.metrics import tie_aware_ranking_metrics + +INPUT_COLUMNS = ("patient_id", "peptide", "hla", "score", "model") +KS = (5, 20) +CORE_METRICS = ("recall@5", "recall@20", "ndcg@5", "mrr") +BOOTSTRAP_REPLICATES = 1000 +BOOTSTRAP_SEED = 20260820 + +Key = tuple[str, str, str] + + +class AuditError(ValueError): + """Raised when an audit cannot be performed without guessing.""" + + +@dataclass(frozen=True) +class BenchmarkSpec: + name: str + benchmark_path: str + overlap_path: str + + +@dataclass(frozen=True) +class BenchmarkRecord: + dataset: str + patient_id: str + peptide: str + hla: str + label: int + record_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class Catalog: + records: dict[Key, BenchmarkRecord] + dataset_keys: dict[str, frozenset[Key]] + overlap_rows: dict[str, dict[str, dict[str, str]]] + collapsed_duplicate_rows: dict[str, int] + + +SPECS = ( + BenchmarkSpec( + "TESLA", + "data/processed/benchmark.csv", + "research/training_overlap_audit.csv", + ), + BenchmarkSpec( + "IMPROVE", + "data/processed/improve_benchmark_full.csv", + "research/training_overlap_audit_improve.csv", + ), + BenchmarkSpec( + "Zhao2026", + "data/processed/zhao_vaccine_benchmark_full.csv", + "research/training_overlap_audit_zhao.csv", + ), +) + +MODEL_ALIASES = { + "prime": "prime2", + "prime2": "prime2", + "prime20": "prime2", + "bigmhc": "bigmhc", + "bigmhcv10": "bigmhc", + "deepimmuno": "deepimmuno", + "deepimmunocnn": "deepimmuno", + "deephlapan": "deephlapan", + "deephlapan111": "deephlapan", +} + +OVERLAP_FIELDS: dict[str, dict[str, dict[str, str | None]]] = { + "TESLA": { + "prime2": { + "exact": "exact_peptide_hla_in_prime2_train", + "peptide_only": "peptide_in_prime2_train", + "near": None, + }, + "bigmhc": { + "exact": "exact_bigmhc_im_trainval_overlap", + "peptide_only": None, + "near": None, + }, + }, + "IMPROVE": { + "prime2": { + "exact": "exact_peptide_hla_in_prime2_train", + "peptide_only": "peptide_in_prime2_train", + "near": "near_hamming1_same_hla_prime2_train", + }, + "bigmhc": { + "exact": "exact_bigmhc_im_trainval_overlap", + "peptide_only": None, + "near": None, + }, + }, + "Zhao2026": { + "prime2": { + "exact": "exact_prime2_peptide_hla", + "peptide_only": "peptide_only_prime2_different_hla", + "near": "near_hamming1_prime2_same_hla", + }, + "bigmhc": { + "exact": "exact_bigmhc_im_trainval", + "peptide_only": None, + "near": None, + }, + "deepimmuno": { + "exact": "exact_deepimmuno_peptide_hla", + "peptide_only": "peptide_only_deepimmuno_different_hla", + "near": "near_hamming1_deepimmuno_same_hla", + }, + }, +} + + +def _read_csv(path: Path) -> tuple[list[str], list[dict[str, str]]]: + if not path.exists(): + raise AuditError(f"missing required frozen artifact: {path}") + with path.open(newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + if reader.fieldnames is None: + raise AuditError(f"CSV has no header: {path}") + return reader.fieldnames, list(reader) + + +def _key(patient_id: str, peptide: str, hla: str) -> Key: + return patient_id.strip(), peptide.strip().upper(), hla.strip() + + +def load_catalog(root: Path) -> Catalog: + grouped: dict[Key, list[tuple[str, dict[str, str]]]] = defaultdict(list) + overlap_rows: dict[str, dict[str, dict[str, str]]] = {} + for spec in SPECS: + _header, rows = _read_csv(root / spec.benchmark_path) + for row in rows: + grouped[_key(row["patient_id"], row["peptide"], row["hla"])].append( + (spec.name, row) + ) + _overlap_header, audit_rows = _read_csv(root / spec.overlap_path) + overlap_by_id = {row["record_id"]: row for row in audit_rows} + if len(overlap_by_id) != len(audit_rows): + raise AuditError(f"duplicate record_id in {spec.overlap_path}") + overlap_rows[spec.name] = overlap_by_id + + records: dict[Key, BenchmarkRecord] = {} + dataset_keys: dict[str, set[Key]] = defaultdict(set) + collapsed: dict[str, int] = defaultdict(int) + for key, members in grouped.items(): + datasets = {dataset for dataset, _row in members} + if len(datasets) != 1: + raise AuditError(f"ambiguous cross-dataset benchmark key: {key}") + dataset = next(iter(datasets)) + labels = {row["immunogenicity"] for _dataset, row in members} + if not labels <= {"0", "1"} or len(labels) != 1: + raise AuditError(f"conflicting or unknown labels for benchmark key: {key}") + record_ids = tuple(sorted(row["record_id"] for _dataset, row in members)) + if any(record_id not in overlap_rows[dataset] for record_id in record_ids): + raise AuditError(f"missing training-overlap row for benchmark key: {key}") + records[key] = BenchmarkRecord( + dataset=dataset, + patient_id=key[0], + peptide=key[1], + hla=key[2], + label=int(next(iter(labels))), + record_ids=record_ids, + ) + dataset_keys[dataset].add(key) + collapsed[dataset] += len(members) - 1 + return Catalog( + records=records, + dataset_keys={name: frozenset(keys) for name, keys in dataset_keys.items()}, + overlap_rows=overlap_rows, + collapsed_duplicate_rows=dict(collapsed), + ) + + +def _model_family(model: str) -> str | None: + normalized = re.sub(r"[^a-z0-9]+", "", model.lower()) + return MODEL_ALIASES.get(normalized) + + +def _truthy_overlap(value: str) -> bool: + return value.strip() == "1" + + +def _record_has_overlap( + record: BenchmarkRecord, + overlap_rows: dict[str, dict[str, dict[str, str]]], + field: str, +) -> bool: + return any( + _truthy_overlap(overlap_rows[record.dataset][record_id].get(field, "")) + for record_id in record.record_ids + ) + + +def _leakage_for_model( + model: str, + supported: set[Key], + datasets: set[str], + catalog: Catalog, +) -> tuple[dict[str, object], set[Key]]: + family = _model_family(model) + if family is None: + return ( + { + "risk": "unknown_training_reference", + "recognized_training_reference": False, + "exact_overlap_records": None, + "peptide_only_overlap_records": None, + "near_overlap_records": None, + "checked_datasets": [], + "unknown_datasets": sorted(datasets), + }, + set(), + ) + + exact_keys: set[Key] = set() + peptide_only_keys: set[Key] = set() + near_keys: set[Key] = set() + exact_checked: set[str] = set() + peptide_checked: set[str] = set() + near_checked: set[str] = set() + unknown_datasets: set[str] = set() + for dataset in datasets: + fields = OVERLAP_FIELDS.get(dataset, {}).get(family) + if fields is None or fields["exact"] is None: + unknown_datasets.add(dataset) + continue + exact_checked.add(dataset) + if fields["peptide_only"] is not None: + peptide_checked.add(dataset) + if fields["near"] is not None: + near_checked.add(dataset) + for key in supported & set(catalog.dataset_keys[dataset]): + record = catalog.records[key] + has_exact_overlap = _record_has_overlap( + record, catalog.overlap_rows, str(fields["exact"]) + ) + if has_exact_overlap: + exact_keys.add(key) + peptide_field = fields["peptide_only"] + if ( + peptide_field + and not has_exact_overlap + and _record_has_overlap(record, catalog.overlap_rows, peptide_field) + ): + peptide_only_keys.add(key) + near_field = fields["near"] + if near_field and _record_has_overlap(record, catalog.overlap_rows, near_field): + near_keys.add(key) + + if exact_keys: + risk = "high_exact_overlap" + elif near_keys or peptide_only_keys: + risk = "possible_sequence_overlap" + elif unknown_datasets: + risk = "partially_unknown" + else: + risk = "no_known_overlap_in_checked_dimensions" + return ( + { + "risk": risk, + "recognized_training_reference": True, + "model_family": family, + "exact_overlap_records": len(exact_keys) if exact_checked else None, + "peptide_only_overlap_records": ( + len(peptide_only_keys) if peptide_checked else None + ), + "near_overlap_records": len(near_keys) if near_checked else None, + "checked_datasets": sorted(exact_checked), + "unknown_datasets": sorted(unknown_datasets), + "excluded_from_metrics_as_exact_overlap": len(exact_keys), + }, + exact_keys, + ) + + +def _percentile(values: list[float], probability: float) -> float: + ordered = sorted(values) + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _patient_metric_report( + keys: set[Key], + scores: dict[Key, float | None], + catalog: Catalog, +) -> dict[str, object]: + if not keys: + return { + "status": "no_leakage_filtered_common_support", + "positive_bearing_patients": 0, + "metrics": {}, + } + by_patient: dict[str, list[Key]] = defaultdict(list) + for key in sorted(keys): + by_patient[catalog.records[key].patient_id].append(key) + patient_values: list[dict[str, float]] = [] + for patient_keys in by_patient.values(): + labels = [catalog.records[key].label for key in patient_keys] + if not sum(labels): + continue + numeric_scores = [scores[key] for key in patient_keys] + if any(score is None for score in numeric_scores): + raise AssertionError("common-support keys must have numeric scores") + patient_values.append( + tie_aware_ranking_metrics( + labels, + [float(score) for score in numeric_scores], + KS, + ) + ) + if not patient_values: + return { + "status": "no_positive_bearing_patients", + "positive_bearing_patients": 0, + "metrics": {}, + } + + rng = random.Random(BOOTSTRAP_SEED) + samples: dict[str, list[float]] = {metric: [] for metric in CORE_METRICS} + for _ in range(BOOTSTRAP_REPLICATES): + draw = rng.choices(patient_values, k=len(patient_values)) + for metric in CORE_METRICS: + samples[metric].append(mean(values[metric] for values in draw)) + metrics = {} + for metric in CORE_METRICS: + values = samples[metric] + metrics[metric] = { + "estimate": mean(patient[metric] for patient in patient_values), + "ci95": { + "low": _percentile(values, 0.025), + "high": _percentile(values, 0.975), + }, + } + return { + "status": "ok", + "positive_bearing_patients": len(patient_values), + "bootstrap_unit": "patient", + "bootstrap_replicates": BOOTSTRAP_REPLICATES, + "seed": BOOTSTRAP_SEED, + "metrics": metrics, + } + + +def audit_predictions(path: Path, root: Path) -> dict[str, object]: + """Return a deterministic audit report for a five-column prediction CSV.""" + header, rows = _read_csv(path) + if len(header) != len(INPUT_COLUMNS) or set(header) != set(INPUT_COLUMNS): + raise AuditError( + "prediction CSV header must contain exactly: " + ", ".join(INPUT_COLUMNS) + ) + if not rows: + raise AuditError("prediction CSV has no data rows") + + catalog = load_catalog(root) + predictions: dict[str, dict[Key, float | None]] = defaultdict(dict) + inferred_datasets: set[str] = set() + for line_number, row in enumerate(rows, start=2): + model = row["model"].strip() + key = _key(row["patient_id"], row["peptide"], row["hla"]) + if not model or not all(key): + raise AuditError(f"line {line_number}: patient_id, peptide, hla and model are required") + if key not in catalog.records: + raise AuditError(f"line {line_number}: key not found in a frozen benchmark: {key}") + if key in predictions[model]: + raise AuditError(f"line {line_number}: duplicate model/benchmark key for {model}: {key}") + score_text = row["score"].strip() + score = None + if score_text: + try: + score = float(score_text) + except ValueError as error: + raise AuditError(f"line {line_number}: score is not numeric") from error + if not math.isfinite(score): + raise AuditError(f"line {line_number}: score must be finite") + predictions[model][key] = score + inferred_datasets.add(catalog.records[key].dataset) + + expected_keys = set().union( + *(set(catalog.dataset_keys[dataset]) for dataset in inferred_datasets) + ) + models = sorted(predictions) + support_by_model: dict[str, set[Key]] = {} + support_report = {} + for model in models: + submitted = set(predictions[model]) + supported = {key for key, score in predictions[model].items() if score is not None} + support_by_model[model] = supported + support_report[model] = { + "expected_records": len(expected_keys), + "submitted_records": len(submitted), + "supported_records": len(supported), + "blank_score_records": len(submitted - supported), + "omitted_records": len(expected_keys - submitted), + "coverage": len(supported) / len(expected_keys), + } + + raw_common = set.intersection(*(support_by_model[model] for model in models)) + leakage_report = {} + exact_exclusions = {} + for model in models: + leakage_report[model], exact_exclusions[model] = _leakage_for_model( + model, + support_by_model[model], + inferred_datasets, + catalog, + ) + audited_support = { + model: support_by_model[model] - exact_exclusions[model] for model in models + } + audited_common = set.intersection(*(audited_support[model] for model in models)) + + patient_metrics = { + model: _patient_metric_report(audited_common, predictions[model], catalog) + for model in models + } + return { + "schema_version": 1, + "input": { + "path": str(path), + "columns": list(INPUT_COLUMNS), + "score_direction": "higher_is_better", + "rows": len(rows), + "models": models, + }, + "benchmark": { + "datasets": sorted(inferred_datasets), + "expected_unique_patient_peptide_hla_records": len(expected_keys), + "collapsed_duplicate_source_rows": sum( + catalog.collapsed_duplicate_rows.get(dataset, 0) + for dataset in inferred_datasets + ), + }, + "leakage": leakage_report, + "support": { + "by_model": support_report, + "raw_common_support": { + "records": len(raw_common), + "coverage": len(raw_common) / len(expected_keys), + }, + "leakage_filtered_common_support": { + "records": len(audited_common), + "coverage": len(audited_common) / len(expected_keys), + }, + }, + "patient_metrics": { + "basis": "leakage-filtered common support; pMHC ranking unit", + "models": patient_metrics, + }, + "limitations": [ + "Only exact model aliases with frozen training references receive checked leakage labels.", + "Unknown training overlap remains unknown; absence of a known match is not proof of independence.", + "Omitted input rows are counted as unsupported because the five-column contract has no status field.", + ], + } diff --git a/src/neorepro/cli.py b/src/neorepro/cli.py index a0cfdae..7e7bd0c 100644 --- a/src/neorepro/cli.py +++ b/src/neorepro/cli.py @@ -4,8 +4,11 @@ import argparse import csv +import json from pathlib import Path +from neorepro.audit import AuditError, audit_predictions + def project_root() -> Path: return Path.cwd() @@ -42,15 +45,30 @@ def main(argv: list[str] | None = None) -> int: subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser("list-predictors") subparsers.add_parser("status") + audit_parser = subparsers.add_parser( + "audit", + help="audit external prediction scores against frozen benchmark evidence", + description=( + "Audit a CSV containing exactly patient_id, peptide, hla, score, model. " + "Scores must be numeric or blank for unsupported rows; higher is better." + ), + ) + audit_parser.add_argument("predictions", type=Path, help="five-column prediction CSV") args = parser.parse_args(argv) root = project_root() if args.command == "list-predictors": return list_predictors(root) if args.command == "status": return status(root) + if args.command == "audit": + try: + report = audit_predictions(args.predictions, root) + except AuditError as error: + parser.error(str(error)) + print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) + return 0 raise AssertionError(args.command) if __name__ == "__main__": raise SystemExit(main()) - diff --git a/src/neorepro/metrics.py b/src/neorepro/metrics.py index c1338cb..e24efdd 100644 --- a/src/neorepro/metrics.py +++ b/src/neorepro/metrics.py @@ -4,6 +4,7 @@ import math from collections.abc import Iterable, Sequence +from math import comb def _validate(labels: Sequence[int], scores: Sequence[float]) -> None: @@ -82,3 +83,70 @@ def ranking_metrics(labels_in_rank_order: Sequence[int], ks: Iterable[int]) -> d idcg = sum(1 / math.log2(rank + 1) for rank in range(1, ideal_hits + 1)) result[f"ndcg@{k}"] = dcg / idcg return result + + +def tie_aware_ranking_metrics( + labels: Sequence[int], scores: Sequence[float], ks: Iterable[int] +) -> dict[str, float]: + """Compute expected patient-ranking metrics over every tied-score ordering.""" + _validate(labels, scores) + positives = sum(labels) + if not positives: + raise ValueError("ranking metrics require a positive-bearing patient") + ordered = sorted(range(len(scores)), key=scores.__getitem__, reverse=True) + score_groups: list[tuple[int, int]] = [] + start = 0 + while start < len(ordered): + end = start + 1 + while end < len(ordered) and scores[ordered[end]] == scores[ordered[start]]: + end += 1 + score_groups.append((end - start, sum(labels[index] for index in ordered[start:end]))) + start = end + + offset = 0 + expected_mrr = 0.0 + for size, group_positives in score_groups: + if group_positives: + denominator = comb(size, group_positives) + expected_mrr = sum( + (comb(size - first, group_positives - 1) / denominator) / (offset + first) + for first in range(1, size - group_positives + 2) + ) + break + offset += size + + result = {"mrr": expected_mrr} + for k in ks: + if k <= 0: + raise ValueError("K must be positive") + limit = min(k, len(ordered)) + remaining = limit + offset = 0 + expected_hits = 0.0 + expected_dcg = 0.0 + zero_hit_probability = 1.0 + for size, group_positives in score_groups: + if not remaining: + break + selected = min(remaining, size) + expected_hits += selected * group_positives / size + expected_dcg += (group_positives / size) * sum( + 1 / math.log2(rank + 1) + for rank in range(offset + 1, offset + selected + 1) + ) + if selected == size: + if group_positives: + zero_hit_probability = 0.0 + elif zero_hit_probability and group_positives: + zero_hit_probability *= comb(size - group_positives, selected) / comb( + size, selected + ) + remaining -= selected + offset += selected + result[f"recall@{k}"] = expected_hits / positives + result[f"precision@{k}"] = expected_hits / limit + result[f"hitrate@{k}"] = 1 - zero_hit_probability + ideal_hits = min(positives, limit) + idcg = sum(1 / math.log2(rank + 1) for rank in range(1, ideal_hits + 1)) + result[f"ndcg@{k}"] = expected_dcg / idcg + return result diff --git a/tests/test_audit_cli.py b/tests/test_audit_cli.py new file mode 100644 index 0000000..c9c8431 --- /dev/null +++ b/tests/test_audit_cli.py @@ -0,0 +1,111 @@ +import csv +import json +from pathlib import Path + +import pytest + +from neorepro.audit import AuditError, audit_predictions +from neorepro.cli import main + +ROOT = Path(__file__).resolve().parents[1] +TESLA = ROOT / "data/processed/benchmark.csv" +ZHAO = ROOT / "data/processed/zhao_vaccine_benchmark_full.csv" +FIELDS = ["patient_id", "peptide", "hla", "score", "model"] + + +def read_rows(path: Path) -> list[dict[str, str]]: + with path.open(newline="", encoding="utf-8-sig") as handle: + return list(csv.DictReader(handle)) + + +def write_predictions(path: Path, rows: list[dict[str, str]]) -> None: + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def prediction(source: dict[str, str], score: str, model: str) -> dict[str, str]: + return { + "patient_id": source["patient_id"], + "peptide": source["peptide"], + "hla": source["hla"], + "score": score, + "model": model, + } + + +def test_audit_reports_leakage_support_and_patient_ci(tmp_path: Path) -> None: + source = read_rows(TESLA)[:3] + path = tmp_path / "predictions.csv" + write_predictions( + path, + [ + prediction(source[0], "0.1", "External A"), + prediction(source[1], "0.2", "External A"), + prediction(source[2], "0.9", "External A"), + prediction(source[0], "0.1", "External B"), + prediction(source[1], "", "External B"), + prediction(source[2], "0.8", "External B"), + ], + ) + + report = audit_predictions(path, ROOT) + + assert report["benchmark"]["datasets"] == ["TESLA"] + assert report["benchmark"]["expected_unique_patient_peptide_hla_records"] == 520 + assert report["leakage"]["External A"]["risk"] == "unknown_training_reference" + assert report["support"]["by_model"]["External A"]["supported_records"] == 3 + assert report["support"]["by_model"]["External B"]["blank_score_records"] == 1 + assert report["support"]["raw_common_support"]["records"] == 2 + metric = report["patient_metrics"]["models"]["External A"]["metrics"]["ndcg@5"] + assert metric["estimate"] == pytest.approx(1.0) + assert metric["ci95"] == {"low": pytest.approx(1.0), "high": pytest.approx(1.0)} + + +def test_known_exact_overlap_is_excluded_from_metrics(tmp_path: Path) -> None: + source = read_rows(TESLA)[:3] + path = tmp_path / "prime.csv" + write_predictions( + path, + [prediction(row, str(index), "PRIME-2.0") for index, row in enumerate(source)], + ) + + report = audit_predictions(path, ROOT) + + leakage = report["leakage"]["PRIME-2.0"] + assert leakage["risk"] == "high_exact_overlap" + assert leakage["exact_overlap_records"] == 3 + assert leakage["excluded_from_metrics_as_exact_overlap"] == 3 + assert report["support"]["leakage_filtered_common_support"]["records"] == 0 + assert report["patient_metrics"]["models"]["PRIME-2.0"]["metrics"] == {} + + +def test_five_column_key_collapses_concordant_source_duplicates(tmp_path: Path) -> None: + source = read_rows(ZHAO)[0] + path = tmp_path / "zhao.csv" + write_predictions(path, [prediction(source, "0.5", "External")]) + + report = audit_predictions(path, ROOT) + + assert report["benchmark"]["expected_unique_patient_peptide_hla_records"] == 2315 + assert report["benchmark"]["collapsed_duplicate_source_rows"] == 2 + + +def test_audit_rejects_non_contract_header(tmp_path: Path) -> None: + path = tmp_path / "bad.csv" + path.write_text("patient_id,peptide,hla,score,model,label\n", encoding="utf-8") + + with pytest.raises(AuditError, match="header must contain exactly"): + audit_predictions(path, ROOT) + + +def test_cli_prints_json_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source = read_rows(TESLA)[0] + path = tmp_path / "predictions.csv" + write_predictions(path, [prediction(source, "0.5", "External")]) + + assert main(["audit", str(path)]) == 0 + output = json.loads(capsys.readouterr().out) + assert output["input"]["columns"] == FIELDS + assert output["input"]["score_direction"] == "higher_is_better" diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 2ed37ed..bfb01eb 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -2,7 +2,12 @@ import pytest -from neorepro.metrics import auroc, average_precision, ranking_metrics +from neorepro.metrics import ( + auroc, + average_precision, + ranking_metrics, + tie_aware_ranking_metrics, +) def test_perfect_pooled_ranking() -> None: @@ -33,3 +38,10 @@ def test_patient_top_k_uses_available_candidates() -> None: def test_ranking_requires_positive_patient() -> None: with pytest.raises(ValueError, match="positive-bearing"): ranking_metrics([0, 0], [5]) + + +def test_tie_aware_ranking_uses_expected_ordering() -> None: + result = tie_aware_ranking_metrics([1, 0], [0.5, 0.5], [1]) + assert result["recall@1"] == pytest.approx(0.5) + assert result["hitrate@1"] == pytest.approx(0.5) + assert result["mrr"] == pytest.approx(0.75) From 5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1 Mon Sep 17 00:00:00 2001 From: stevezkw <53121066+stevezkw1998@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:23:03 -0700 Subject: [PATCH 4/6] Integrate RCC extension and reusable benchmark contracts --- .gitignore | 2 + FINAL_REPORT.md | 6 + Makefile | 47 +- README.de.md | 7 +- README.es.md | 7 +- README.fr.md | 7 +- README.ja.md | 7 +- README.ko.md | 9 +- README.md | 7 +- README.zh-CN.md | 7 +- README.zh-TW.md | 7 +- contracts/README.md | 13 + contracts/dataset-card.example.json | 10 + contracts/prediction-artifact.schema.json | 11 + contracts/predictor-card.example.json | 12 + contracts/synthetic/README.md | 3 + contracts/synthetic/benchmark.csv | 5 + contracts/synthetic/predictions.csv | 5 + data/predictor_registry.csv | 7 + data/processed/rcc_vaccine_benchmark.csv | 130 + data/processed/rcc_vaccine_benchmark_full.csv | 130 + data/rcc_vaccine_summary.json | 9 + i18n/readme_state.json | 44 +- paper/manuscript_resource.md | 35 +- paper/manuscript_template.md | 14 +- paper/references.bib | 10 + predictors/example/adapter.py | 17 + .../attempts/20260821T065057715285Z-test.json | 16 + .../20260821T065057715285Z-test.stderr.log | 0 .../20260821T065057715285Z-test.stdout.log | 49 + .../attempts/20260821T065145972187Z-run.json | 17 + .../20260821T065145972187Z-run.stderr.log | 3 + .../20260821T065145972187Z-run.stdout.log | 0 predictors/mhcmatch/attempts/clone.stderr.log | 1 + predictors/mhcmatch/attempts/clone.stdout.log | 0 .../mhcmatch/attempts/install.stderr.log | 26 + .../mhcmatch/attempts/install.stdout.log | 0 predictors/mhcmatch/attempts/smoke.stderr.log | 1 + predictors/mhcmatch/attempts/smoke.stdout.log | 1 + .../mhcmatch/attempts/sweep_receipt.json | 12 + predictors/mhcnuggets/adapter.py | 40 + .../20260821T065001940493Z-adapter.json | 22 + .../20260821T065001940493Z-adapter.stderr.log | 2 + .../20260821T065001940493Z-adapter.stdout.log | 7 + .../mhcnuggets/attempts/clone.stderr.log | 105 + .../mhcnuggets/attempts/clone.stdout.log | 0 .../mhcnuggets/attempts/install.stderr.log | 8 + .../mhcnuggets/attempts/install.stdout.log | 0 .../mhcnuggets/attempts/smoke.stderr.log | 0 .../mhcnuggets/attempts/smoke.stdout.log | 1 + .../mhcnuggets/attempts/sweep_receipt.json | 12 + predictors/mhcnuggets/fixture.csv | 4 + predictors/mhcnuggets/fixture_predictions.csv | 4 + .../attempts/20260821T065057715299Z-test.json | 16 + .../20260821T065057715299Z-test.stderr.log | 4 + .../20260821T065057715299Z-test.stdout.log | 47 + predictors/neofox/attempts/clone.stderr.log | 103 + predictors/neofox/attempts/clone.stdout.log | 0 predictors/neofox/attempts/install.stderr.log | 8 + predictors/neofox/attempts/install.stdout.log | 0 predictors/neofox/attempts/smoke.stderr.log | 0 predictors/neofox/attempts/smoke.stdout.log | 1 + predictors/neofox/attempts/sweep_receipt.json | 12 + .../neoguider/attempts/clone.stderr.log | 1 + .../neoguider/attempts/clone.stdout.log | 0 .../neoguider/attempts/install.stderr.log | 5 + .../neoguider/attempts/install.stdout.log | 0 .../neoguider/attempts/smoke.stderr.log | 1 + .../neoguider/attempts/smoke.stdout.log | 1 + .../neoguider/attempts/sweep_receipt.json | 12 + .../pvactools/attempts/clone.stderr.log | 110 + .../pvactools/attempts/clone.stdout.log | 0 .../pvactools/attempts/install.stderr.log | 8 + .../pvactools/attempts/install.stdout.log | 0 .../pvactools/attempts/smoke.stderr.log | 0 .../pvactools/attempts/smoke.stdout.log | 49 + .../pvactools/attempts/sweep_receipt.json | 13 + predictors/seq2neo/attempts/clone.stderr.log | 1 + predictors/seq2neo/attempts/clone.stdout.log | 0 .../seq2neo/attempts/install.stderr.log | 9 + .../seq2neo/attempts/install.stdout.log | 0 predictors/seq2neo/attempts/smoke.stderr.log | 1 + predictors/seq2neo/attempts/smoke.stdout.log | 1 + .../seq2neo/attempts/sweep_receipt.json | 12 + predictors/vaxrank/attempts/clone.stderr.log | 1 + predictors/vaxrank/attempts/clone.stdout.log | 0 .../vaxrank/attempts/install.stderr.log | 106 + .../vaxrank/attempts/install.stdout.log | 0 predictors/vaxrank/attempts/smoke.stderr.log | 0 predictors/vaxrank/attempts/smoke.stdout.log | 1 + .../vaxrank/attempts/sweep_receipt.json | 12 + reports/manuscript_resource_audit.json | 1 + reports/rcc_predictor_run.json | 108 + research/extension_protocol_rcc_v1.json | 46 + .../external_cohort_failure_protocols.json | 72 + research/external_cohort_funnel.csv | 11 + research/research_log.md | 35 + research/training_overlap_audit_rcc.csv | 130 + research/training_overlap_summary_rcc.json | 23 + results/analysis/rcc/metrics.json | 2911 +++++++++++++++++ results/analysis/rcc/missingness.csv | 10 + results/analysis/rcc/paired_differences.csv | 91 + .../analysis/stability/analysis_metadata.json | 68 + .../dataset_predictor_metric_matrix.csv | 61 + .../stability/endpoint_domain_auroc.png | Bin 0 -> 91312 bytes .../stability/endpoint_domain_auroc.svg | 43 + .../stability/endpoint_domain_metadata.csv | 4 + .../stability/leave_one_domain_out.csv | 19 + .../model_selection_first_probability.csv | 13 + results/analysis/stability/rank_stability.csv | 96 + .../stability/sensitivity_summary.csv | 13 + results/manifest.json | 195 +- results/raw_predictions/rcc/bigmhc-v1.0.csv | 130 + .../raw_predictions/rcc/deephlapan-1.1.1.csv | 130 + .../raw_predictions/rcc/deepimmuno-cnn.csv | 130 + results/raw_predictions/rcc/prime-2.0.csv | 130 + scripts/analyze_stability.py | 387 +++ scripts/audit_release_boundary.py | 49 + scripts/build_manuscript.py | 40 +- scripts/build_rcc_benchmark.py | 120 + scripts/build_results_manifest.py | 35 + scripts/evaluate_benchmark.py | 6 +- scripts/render_stability_figure.py | 23 + scripts/reproduce_public_predictors.py | 118 + scripts/validate_external_cohort_funnel.py | 45 + src/neorepro/cli.py | 70 +- src/neorepro/contract.py | 211 ++ tests/test_contract.py | 56 + tests/test_rcc_extension.py | 41 + tests/test_stability_analysis.py | 48 + 130 files changed, 7083 insertions(+), 70 deletions(-) create mode 100644 contracts/README.md create mode 100644 contracts/dataset-card.example.json create mode 100644 contracts/prediction-artifact.schema.json create mode 100644 contracts/predictor-card.example.json create mode 100644 contracts/synthetic/README.md create mode 100644 contracts/synthetic/benchmark.csv create mode 100644 contracts/synthetic/predictions.csv create mode 100644 data/processed/rcc_vaccine_benchmark.csv create mode 100644 data/processed/rcc_vaccine_benchmark_full.csv create mode 100644 data/rcc_vaccine_summary.json create mode 100644 predictors/example/adapter.py create mode 100644 predictors/mhcmatch/attempts/20260821T065057715285Z-test.json create mode 100644 predictors/mhcmatch/attempts/20260821T065057715285Z-test.stderr.log create mode 100644 predictors/mhcmatch/attempts/20260821T065057715285Z-test.stdout.log create mode 100644 predictors/mhcmatch/attempts/20260821T065145972187Z-run.json create mode 100644 predictors/mhcmatch/attempts/20260821T065145972187Z-run.stderr.log create mode 100644 predictors/mhcmatch/attempts/20260821T065145972187Z-run.stdout.log create mode 100644 predictors/mhcmatch/attempts/clone.stderr.log create mode 100644 predictors/mhcmatch/attempts/clone.stdout.log create mode 100644 predictors/mhcmatch/attempts/install.stderr.log create mode 100644 predictors/mhcmatch/attempts/install.stdout.log create mode 100644 predictors/mhcmatch/attempts/smoke.stderr.log create mode 100644 predictors/mhcmatch/attempts/smoke.stdout.log create mode 100644 predictors/mhcmatch/attempts/sweep_receipt.json create mode 100644 predictors/mhcnuggets/adapter.py create mode 100644 predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.json create mode 100644 predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stderr.log create mode 100644 predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stdout.log create mode 100644 predictors/mhcnuggets/attempts/clone.stderr.log create mode 100644 predictors/mhcnuggets/attempts/clone.stdout.log create mode 100644 predictors/mhcnuggets/attempts/install.stderr.log create mode 100644 predictors/mhcnuggets/attempts/install.stdout.log create mode 100644 predictors/mhcnuggets/attempts/smoke.stderr.log create mode 100644 predictors/mhcnuggets/attempts/smoke.stdout.log create mode 100644 predictors/mhcnuggets/attempts/sweep_receipt.json create mode 100644 predictors/mhcnuggets/fixture.csv create mode 100644 predictors/mhcnuggets/fixture_predictions.csv create mode 100644 predictors/neofox/attempts/20260821T065057715299Z-test.json create mode 100644 predictors/neofox/attempts/20260821T065057715299Z-test.stderr.log create mode 100644 predictors/neofox/attempts/20260821T065057715299Z-test.stdout.log create mode 100644 predictors/neofox/attempts/clone.stderr.log create mode 100644 predictors/neofox/attempts/clone.stdout.log create mode 100644 predictors/neofox/attempts/install.stderr.log create mode 100644 predictors/neofox/attempts/install.stdout.log create mode 100644 predictors/neofox/attempts/smoke.stderr.log create mode 100644 predictors/neofox/attempts/smoke.stdout.log create mode 100644 predictors/neofox/attempts/sweep_receipt.json create mode 100644 predictors/neoguider/attempts/clone.stderr.log create mode 100644 predictors/neoguider/attempts/clone.stdout.log create mode 100644 predictors/neoguider/attempts/install.stderr.log create mode 100644 predictors/neoguider/attempts/install.stdout.log create mode 100644 predictors/neoguider/attempts/smoke.stderr.log create mode 100644 predictors/neoguider/attempts/smoke.stdout.log create mode 100644 predictors/neoguider/attempts/sweep_receipt.json create mode 100644 predictors/pvactools/attempts/clone.stderr.log create mode 100644 predictors/pvactools/attempts/clone.stdout.log create mode 100644 predictors/pvactools/attempts/install.stderr.log create mode 100644 predictors/pvactools/attempts/install.stdout.log create mode 100644 predictors/pvactools/attempts/smoke.stderr.log create mode 100644 predictors/pvactools/attempts/smoke.stdout.log create mode 100644 predictors/pvactools/attempts/sweep_receipt.json create mode 100644 predictors/seq2neo/attempts/clone.stderr.log create mode 100644 predictors/seq2neo/attempts/clone.stdout.log create mode 100644 predictors/seq2neo/attempts/install.stderr.log create mode 100644 predictors/seq2neo/attempts/install.stdout.log create mode 100644 predictors/seq2neo/attempts/smoke.stderr.log create mode 100644 predictors/seq2neo/attempts/smoke.stdout.log create mode 100644 predictors/seq2neo/attempts/sweep_receipt.json create mode 100644 predictors/vaxrank/attempts/clone.stderr.log create mode 100644 predictors/vaxrank/attempts/clone.stdout.log create mode 100644 predictors/vaxrank/attempts/install.stderr.log create mode 100644 predictors/vaxrank/attempts/install.stdout.log create mode 100644 predictors/vaxrank/attempts/smoke.stderr.log create mode 100644 predictors/vaxrank/attempts/smoke.stdout.log create mode 100644 predictors/vaxrank/attempts/sweep_receipt.json create mode 100644 reports/rcc_predictor_run.json create mode 100644 research/extension_protocol_rcc_v1.json create mode 100644 research/external_cohort_failure_protocols.json create mode 100644 research/external_cohort_funnel.csv create mode 100644 research/training_overlap_audit_rcc.csv create mode 100644 research/training_overlap_summary_rcc.json create mode 100644 results/analysis/rcc/metrics.json create mode 100644 results/analysis/rcc/missingness.csv create mode 100644 results/analysis/rcc/paired_differences.csv create mode 100644 results/analysis/stability/analysis_metadata.json create mode 100644 results/analysis/stability/dataset_predictor_metric_matrix.csv create mode 100644 results/analysis/stability/endpoint_domain_auroc.png create mode 100644 results/analysis/stability/endpoint_domain_auroc.svg create mode 100644 results/analysis/stability/endpoint_domain_metadata.csv create mode 100644 results/analysis/stability/leave_one_domain_out.csv create mode 100644 results/analysis/stability/model_selection_first_probability.csv create mode 100644 results/analysis/stability/rank_stability.csv create mode 100644 results/analysis/stability/sensitivity_summary.csv create mode 100644 results/raw_predictions/rcc/bigmhc-v1.0.csv create mode 100644 results/raw_predictions/rcc/deephlapan-1.1.1.csv create mode 100644 results/raw_predictions/rcc/deepimmuno-cnn.csv create mode 100644 results/raw_predictions/rcc/prime-2.0.csv create mode 100755 scripts/analyze_stability.py create mode 100755 scripts/audit_release_boundary.py create mode 100755 scripts/build_rcc_benchmark.py create mode 100755 scripts/render_stability_figure.py create mode 100755 scripts/reproduce_public_predictors.py create mode 100755 scripts/validate_external_cohort_funnel.py create mode 100644 src/neorepro/contract.py create mode 100644 tests/test_contract.py create mode 100644 tests/test_rcc_extension.py create mode 100644 tests/test_stability_analysis.py diff --git a/.gitignore b/.gitignore index 0818c8d..529014d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,8 @@ results/raw_predictions/** !results/raw_predictions/improve/baselines/loso/*.csv !results/raw_predictions/zhao/ !results/raw_predictions/zhao/*.csv +!results/raw_predictions/rcc/ +!results/raw_predictions/rcc/*.csv !results/raw_predictions/improve/expanded_9_10/ !results/raw_predictions/improve/expanded_9_10/*.csv results/raw_predictions/improve/peptide_sensitivity/ diff --git a/FINAL_REPORT.md b/FINAL_REPORT.md index b5db565..8da26e7 100644 --- a/FINAL_REPORT.md +++ b/FINAL_REPORT.md @@ -24,6 +24,12 @@ This cohort is complementary, not a pure biological replication. Peptide-pulsed On the original IMPROVE 9–10mer subset, the expanded model set retained the original direction: PRIME AUROC 0.605, BigMHC 0.547, DeepImmuno-CNN 0.527 on its 11,036 supported records, and DeepHLApan 0.508. Thus the external reversal is not explained merely by adding the two older models; it is specifically a dataset/endpoint-domain change. +## RCC cohort and reusable-resource extension + +A second endpoint-distinct vaccine extension adds 129 individually assayed short peptides from nine RCC vaccine recipients. Its protocol was frozen before prediction, its source workbook is checksum-pinned, and no known exact PRIME2, BigMHC-construction or DeepImmuno training overlap was identified. Because the cohort is small, uses source-predicted HLA assignments and measures post-vaccination in-vitro stimulation, its results are descriptive and do not establish natural presentation, untreated immunogenicity, clinical efficacy or a universal predictor order. All quantitative RCC results and uncertainty intervals are generated into `paper/manuscript_resource.md` from `results/analysis/rcc/metrics.json`. + +The repository now also includes an expanded public-artifact census and a machine-validated extension contract for datasets, predictors and standardized prediction artifacts. Heterogeneous or failed tools remain visible in the registry but are not promoted into the peptide–HLA benchmark. Three-domain stability outputs are explicitly exploratory, task-stratified and conditional on observed support. + ## Robustness and interpretation - Excluding 35 peptides seen in PRIME2 training only under another HLA left the main direction unchanged (BigMHC AUROC 0.545; PRIME 0.596). Excluding 18 additional same-HLA, same-length Hamming-distance-one records did likewise (0.546; 0.596). diff --git a/Makefile b/Makefile index 68cb8c1..0beabcc 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ export LC_ALL := en_US.UTF-8 export LANG := en_US.UTF-8 PY := uv run python -.PHONY: install setup-predictors download-improve build-improve audit-improve extension download-zhao build-zhao audit-zhao predict-zhao evaluate-zhao predict-expanded-improve evaluate-expanded-improve \ +.PHONY: install setup-predictors contract-gate release-boundary predictor-census external-funnel rcc evaluate-rcc stability download-improve build-improve audit-improve extension download-zhao build-zhao audit-zhao predict-zhao evaluate-zhao predict-expanded-improve evaluate-expanded-improve \ predict-improve baselines-improve evaluate-improve hla-improve figures manuscript \ sensitivity-improve build-peptide-sensitivity evaluate-peptide-sensitivity \ evaluate-peptide-hla-rank-sensitivity build-fixed-sensitivities \ @@ -14,6 +14,47 @@ PY := uv run python install: uv sync --extra dev --extra analysis +contract-gate: + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli dataset validate "$(CURDIR)/contracts/dataset-card.example.json" + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli predictor validate "$(CURDIR)/contracts/predictor-card.example.json" + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli artifact "$(CURDIR)/contracts/synthetic/predictions.csv" --benchmark "$(CURDIR)/contracts/synthetic/benchmark.csv" + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli evaluate "$(CURDIR)/contracts/synthetic/benchmark.csv" "$(CURDIR)/contracts/synthetic/predictions.csv" \ + --output /tmp/neorepro-contract-evaluation.json --report /tmp/neorepro-contract-evaluation.md + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest pytest "$(CURDIR)/tests/test_contract.py" -q + +release-boundary: + $(PY) scripts/audit_release_boundary.py + +predictor-census: + $(PY) scripts/reproduce_public_predictors.py + +external-funnel: + $(PY) scripts/validate_external_cohort_funnel.py + +rcc: + $(PY) scripts/build_rcc_benchmark.py --input data/raw/rcc_2025_table2.xlsx \ + --output data/processed/rcc_vaccine_benchmark.csv --summary data/rcc_vaccine_summary.json + $(PY) scripts/audit_external_training_overlap.py \ + --benchmark data/processed/rcc_vaccine_benchmark.csv \ + --prime2-archive data/raw/prime2_table_s4.xlsx \ + --output research/training_overlap_audit_rcc.csv \ + --summary research/training_overlap_summary_rcc.json + $(MAKE) evaluate-rcc + +evaluate-rcc: + PYTHONPATH=src $(PY) scripts/evaluate_benchmark.py \ + --benchmark data/processed/rcc_vaccine_benchmark.csv \ + --predictions results/raw_predictions/rcc/*.csv \ + --output-dir results/analysis/rcc --bootstrap 2000 --seed 20260820 + +stability: + $(PY) scripts/analyze_stability.py \ + --benchmark data/processed/improve_benchmark.csv data/processed/zhao_vaccine_benchmark.csv data/processed/rcc_vaccine_benchmark.csv \ + --prediction-dir results/raw_predictions/improve results/raw_predictions/zhao results/raw_predictions/rcc \ + --output-dir results/analysis/stability --bootstrap 2000 --seed 20260820 + $(PY) scripts/render_stability_figure.py + + setup-predictors: $(PY) scripts/setup_predictors.py --accept-academic-licenses @@ -218,7 +259,7 @@ figures: evaluate-improve baselines-improve hla-improve --loso results/analysis/improve/baselines/loso/metrics.json \ --hla results/analysis/improve/hla_sensitivity.csv -manuscript: figures sensitivity-improve +manuscript: figures sensitivity-improve evaluate-rcc stability external-funnel contract-gate $(PY) scripts/build_manuscript.py manifest: manuscript validate-metrics @@ -232,7 +273,7 @@ validate-metrics: evaluate-improve baselines-improve sensitivity-improve reproduce-results: install $(MAKE) verify-reproduction -verify-reproduction: manifest +verify-reproduction: manifest release-boundary $(MAKE) test # Also download source data and install/run licensed third-party predictors. diff --git a/README.de.md b/README.de.md index b7dbc9d..e0c00a0 100644 --- a/README.de.md +++ b/README.de.md @@ -15,6 +15,7 @@ Der Beitrag ist eine Benchmark- und Forschungsressource, kein neuer Prädiktor u - **Aktuelles Manuskript:** [als Ressource positioniertes Manuskript](paper/manuscript_resource.md). - **Evidenz in zwei Minuten:** [zweisprachige Expertenübersicht](output/pdf/neorepro_expert_brief_bilingual.pdf) und [Zusammenfassung der unabhängigen Kohortenerweiterung](reports/extension_summary.md). - **Festgeschriebene Ergebnisse reproduzieren:** den Befehl im Abschnitt „Reproduktion“ verwenden. +- **Drittanbieter-Datensatz oder -Prädiktor hinzufügen:** den [plug-in contract](contracts/README.md) verwenden. - **Feste Version zitieren:** [CITATION.cff](CITATION.cff) und [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). Der wissenschaftliche Vertrag und Umfang stehen in [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -22,10 +23,12 @@ Der wissenschaftliche Vertrag und Umfang stehen in [RESEARCH_SPEC.md](RESEARCH_S ## Status - Aktuelle Literaturprüfung: abgeschlossen; Entscheidung `RESCOPE, then GO` -- Reproduzierte Prädiktoren: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN und DeepHLApan +- Benchmark-Prädiktoren: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN und DeepHLApan; für sieben weitere öffentliche Werkzeuge liegen versionierte Profil-, Nichtvergleichbarkeits- oder Fehlschlagsnachweise vor - TESLA-Pilot: abgeschlossen; als Positivkontrolle für Trainingsdatenüberschneidung neu eingestuft - Primärer Benchmark: IMPROVE, 17,475 nach Leakage-Filterung verbleibende Datensätze, 70 Patienten, 3 Kohorten - Primäre IMPROVE-Inferenz: abgeschlossen; 52,425 Vorhersagen fester Werkzeuge ohne fehlende Datensätze +- Externe Domänen: Zhao-Impfkohorte sowie eine separat festgeschriebene RCC-Impfkohorte mit 129 Datensätzen und 9 Patienten +- Wiederverwendbare Erweiterungsschnittstelle: maschinell validierte Dataset Cards, Predictor Cards und Verträge für Vorhersageartefakte - Manuskript: [Ressourcenfassung](paper/manuscript_resource.md), aus festgeschriebenen Ergebnisdateien erzeugt; unabhängige statistische und biologische Prüfung abgeschlossen ## Hauptergebnis @@ -48,7 +51,7 @@ Unabhängige bootstrap-Analysen werden von Make parallelisiert. Bei begrenzter C - **Prüfbare Ausgaben:** [endgültige Ergebnistabelle](results/final_results.csv), [Abbildungen](results/figures/), [Prüfung auf Trainingsüberschneidungen](research/training_overlap_summary_improve.json) und [SHA-256-Manifest](results/manifest.json). - **Publikationsplanung:** [Strategie für Zielzeitschriften](reports/target_venues_2026-08-20.md). -Die unabhängige Erweiterung um die Zhao 2026-Impfkohorte lässt sich mit `make -j4 extension` reproduzieren. Die kompakte Evidenzübersicht steht in [reports/extension_summary.md](reports/extension_summary.md), der vor der Inferenz festgeschriebene Studienvertrag in [research/extension_protocol.json](research/extension_protocol.json). Der externe Endpunkt ist ELISPOT nach der Impfung und Verabreichung peptidgepulster dendritischer Zellen; er darf nicht als natürliche Tumorpräsentation oder klinische Wirksamkeit interpretiert werden. +Die unabhängige Erweiterung um die Zhao 2026-Impfkohorte lässt sich mit `make -j4 extension` reproduzieren. Die kompakte Evidenzübersicht steht in [reports/extension_summary.md](reports/extension_summary.md), der vor der Inferenz festgeschriebene Studienvertrag in [research/extension_protocol.json](research/extension_protocol.json). Die separat festgeschriebene RCC-Erweiterung steht in [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), explorative Stabilitätsausgaben für drei Domänen unter `results/analysis/stability/`. Beide externen Endpunkte wurden nach der Impfung erhoben und dürfen nicht als natürliche Tumorpräsentation oder klinische Wirksamkeit interpretiert werden. ## Lizenz diff --git a/README.es.md b/README.es.md index 9aff206..604ecf2 100644 --- a/README.es.md +++ b/README.es.md @@ -15,6 +15,7 @@ Es una contribución de referencia y recursos, no un predictor nuevo ni una afir - **Manuscrito actual:** [manuscrito orientado como recurso](paper/manuscript_resource.md). - **Resumen de la evidencia en dos minutos:** [informe bilingüe para expertos](output/pdf/neorepro_expert_brief_bilingual.pdf) y [resumen de la extensión con cohorte independiente](reports/extension_summary.md). - **Reproducir los resultados fijados:** usar el comando de la sección «Reproducción». +- **Añadir un conjunto de datos o predictor de terceros:** consultar el [plug-in contract](contracts/README.md). - **Citar una versión fija:** [CITATION.cff](CITATION.cff) y [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). El contrato científico y el alcance están en [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -22,10 +23,12 @@ El contrato científico y el alcance están en [RESEARCH_SPEC.md](RESEARCH_SPEC. ## Estado - Auditoría de la literatura actual: completa; decisión `RESCOPE, then GO` -- Predictores reproducidos: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN y DeepHLApan +- Predictores evaluados: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN y DeepHLApan; otros siete instrumentos públicos conservan registros versionados de perfil, no comparabilidad o fallo de reproducción - Piloto TESLA: completo; reclasificado como control positivo de solapamiento con entrenamiento - Evaluación principal: IMPROVE, 17,475 registros tras filtrar fugas, 70 pacientes y 3 cohortes - Inferencia principal de IMPROVE: completa; 52,425 predicciones de herramientas fijas sin registros ausentes +- Dominios externos: cohorte de vacunación Zhao y una cohorte RCC fijada por separado con 129 registros y 9 pacientes +- Interfaz de extensión reutilizable: Dataset Cards, Predictor Cards y contratos de artefactos de predicción validados por máquina - Manuscrito: [versión orientada como recurso](paper/manuscript_resource.md), generada a partir de resultados fijados; revisión estadística y biológica independiente completada ## Resultado principal @@ -48,7 +51,7 @@ Make paraleliza los análisis bootstrap independientes. Use `make reproduce-resu - **Resultados auditables:** [tabla final de resultados](results/final_results.csv), [figuras](results/figures/), [auditoría de solapamiento con entrenamiento](research/training_overlap_summary_improve.json) y [manifiesto SHA-256](results/manifest.json). - **Planificación del envío:** [estrategia de revistas objetivo](reports/target_venues_2026-08-20.md). -La extensión independiente con la cohorte de vacunación Zhao 2026 se reproduce con `make -j4 extension`. El resumen conciso de la evidencia está en [reports/extension_summary.md](reports/extension_summary.md), y el contrato congelado antes de la inferencia en [research/extension_protocol.json](research/extension_protocol.json). El criterio externo es ELISPOT posterior a la vacunación tras administrar células dendríticas pulsadas con péptidos; no debe interpretarse como presentación tumoral natural ni eficacia clínica. +La extensión independiente con la cohorte de vacunación Zhao 2026 se reproduce con `make -j4 extension`. El resumen conciso de la evidencia está en [reports/extension_summary.md](reports/extension_summary.md), y el contrato congelado antes de la inferencia en [research/extension_protocol.json](research/extension_protocol.json). La extensión RCC fijada por separado está en [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), y las salidas exploratorias de estabilidad de tres dominios en `results/analysis/stability/`. Ambos criterios externos son posteriores a la vacunación y no deben interpretarse como presentación tumoral natural ni eficacia clínica. ## Licencia diff --git a/README.fr.md b/README.fr.md index 1a187be..87de1c9 100644 --- a/README.fr.md +++ b/README.fr.md @@ -15,6 +15,7 @@ Il s’agit d’une contribution de type ressource et benchmark, et non d’un n - **Manuscrit actuel :** [manuscrit positionné comme ressource](paper/manuscript_resource.md). - **Synthèse des preuves en deux minutes :** [note bilingue pour experts](output/pdf/neorepro_expert_brief_bilingual.pdf) et [résumé de l’extension sur une cohorte indépendante](reports/extension_summary.md). - **Reproduire les résultats figés :** utiliser la commande de la section « Reproduction » ci-dessous. +- **Ajouter un jeu de données ou un prédicteur tiers :** consulter le [plug-in contract](contracts/README.md). - **Citer une version figée :** [CITATION.cff](CITATION.cff) et [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). Le contrat scientifique et le périmètre sont décrits dans [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -22,10 +23,12 @@ Le contrat scientifique et le périmètre sont décrits dans [RESEARCH_SPEC.md]( ## État - Audit de la littérature actuelle : terminé ; décision `RESCOPE, then GO` -- Prédicteurs reproduits : MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN et DeepHLApan +- Prédicteurs évalués : MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN et DeepHLApan ; sept autres outils publics disposent de dossiers versionnés de profil, de non-comparabilité ou d’échec de reproduction - Pilote TESLA : terminé ; reclassé comme contrôle positif de chevauchement avec l’entraînement - Benchmark principal : IMPROVE, 17,475 enregistrements après filtrage des fuites, 70 patients et 3 cohortes - Inférence IMPROVE principale : terminée ; 52,425 prédictions d’outils fixes sans enregistrement manquant +- Domaines externes : cohorte vaccinale Zhao et cohorte vaccinale RCC figée séparément avec 129 enregistrements et 9 patients +- Interface d’extension réutilisable : Dataset Cards, Predictor Cards et contrats d’artefacts de prédiction validés automatiquement - Manuscrit : [version positionnée comme ressource](paper/manuscript_resource.md), générée à partir des résultats figés ; revues statistique et biologique indépendantes terminées ## Résultat principal @@ -48,7 +51,7 @@ Make parallélise les analyses bootstrap indépendantes. Utilisez `make reproduc - **Sorties auditables :** [table finale des résultats](results/final_results.csv), [figures](results/figures/), [audit du chevauchement avec l’entraînement](research/training_overlap_summary_improve.json) et [manifeste SHA-256](results/manifest.json). - **Planification de la soumission :** [stratégie des revues cibles](reports/target_venues_2026-08-20.md). -L’extension indépendante sur la cohorte vaccinale Zhao 2026 est reproduite avec `make -j4 extension`. La synthèse concise des preuves se trouve dans [reports/extension_summary.md](reports/extension_summary.md), et le contrat figé avant l’inférence dans [research/extension_protocol.json](research/extension_protocol.json). Le critère externe est un ELISPOT après vaccination et administration de cellules dendritiques chargées en peptides ; il ne doit pas être interprété comme une présentation tumorale naturelle ni comme une efficacité clinique. +L’extension indépendante sur la cohorte vaccinale Zhao 2026 est reproduite avec `make -j4 extension`. La synthèse concise des preuves se trouve dans [reports/extension_summary.md](reports/extension_summary.md), et le contrat figé avant l’inférence dans [research/extension_protocol.json](research/extension_protocol.json). L’extension RCC figée séparément se trouve dans [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), et les sorties exploratoires de stabilité sur trois domaines dans `results/analysis/stability/`. Les deux critères externes sont postérieurs à la vaccination et ne doivent pas être interprétés comme une présentation tumorale naturelle ni comme une efficacité clinique. ## Licence diff --git a/README.ja.md b/README.ja.md index 5ad42af..d8835d0 100644 --- a/README.ja.md +++ b/README.ja.md @@ -15,6 +15,7 @@ NeoRepro は、公開されている MHC-I ペプチド–HLA ネオアンチゲ - **現行原稿：**[リソースとして位置付けた原稿](paper/manuscript_resource.md)。 - **短時間で読めるエビデンス概要：**[中英併記の専門家向け概要](output/pdf/neorepro_expert_brief_bilingual.pdf)および[独立コホート拡張の概要](reports/extension_summary.md)。 - **凍結済み結果の再現：**下記「再現」節のコマンドを使用してください。 +- **第三者データセットまたは予測器の追加：**[plug-in contract](contracts/README.md) を参照してください。 - **固定版の引用：**[CITATION.cff](CITATION.cff) および [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)を参照してください。 科学的な研究契約と範囲については [RESEARCH_SPEC.md](RESEARCH_SPEC.md) を参照してください。 @@ -22,10 +23,12 @@ NeoRepro は、公開されている MHC-I ペプチド–HLA ネオアンチゲ ## 状況 - 最新文献監査：完了、判断は `RESCOPE, then GO` -- 再現済み予測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN、DeepHLApan +- ベンチマーク対象予測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN、DeepHLApan。さらに七つの公開ツールについて、プロファイル限定、比較不能、または再現失敗の記録をバージョン管理 - TESLA パイロット：完了、学習データ重複の陽性対照として再分類 - 主要ベンチマーク：IMPROVE、リーケージ除外後 17,475 レコード、70 患者、3 コホート - 主要 IMPROVE 推論：完了、固定ツールによる 52,425 予測、欠損レコードなし +- 外部ドメイン：Zhao ワクチンコホートと、別途凍結した 129 レコード・9 患者の RCC ワクチンコホート +- 再利用可能な拡張インターフェース：機械検証済み Dataset Card、Predictor Card、予測アーティファクト契約 - 原稿：[リソース位置付け版](paper/manuscript_resource.md)、凍結結果ファイルから生成、独立した統計学・生物学レビュー済み ## 主な結果 @@ -48,7 +51,7 @@ make -j4 reproduce-results - **監査可能な出力：**[最終結果表](results/final_results.csv)、[図](results/figures/)、[学習データ重複監査](research/training_overlap_summary_improve.json)、[SHA-256 マニフェスト](results/manifest.json)。 - **投稿計画：**[投稿先戦略](reports/target_venues_2026-08-20.md)。 -独立した Zhao 2026 ワクチンコホート拡張は `make -j4 extension` で再現できます。簡潔なエビデンス概要は [reports/extension_summary.md](reports/extension_summary.md)、推論前に凍結した研究契約は [research/extension_protocol.json](research/extension_protocol.json) にあります。外部エンドポイントは、ペプチドをパルスした樹状細胞投与後のワクチン接種後 ELISPOT であり、自然な腫瘍提示や臨床効果を示すものではありません。 +独立した Zhao 2026 ワクチンコホート拡張は `make -j4 extension` で再現できます。簡潔なエビデンス概要は [reports/extension_summary.md](reports/extension_summary.md)、推論前に凍結した研究契約は [research/extension_protocol.json](research/extension_protocol.json) にあります。別途凍結した RCC 拡張は [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json)、三ドメインの探索的安定性出力は `results/analysis/stability/` にあります。両外部エンドポイントはワクチン接種後の測定であり、自然な腫瘍提示や臨床効果を示すものではありません。 ## ライセンス diff --git a/README.ko.md b/README.ko.md index d7b4057..40df0b7 100644 --- a/README.ko.md +++ b/README.ko.md @@ -15,6 +15,7 @@ NeoRepro는 공개 MHC-I 펩타이드–HLA 신생항원 예측기를 위한 데 - **현재 원고:** [리소스 중심 원고](paper/manuscript_resource.md). - **간단한 증거 요약:** [중영 이중언어 전문가 요약](output/pdf/neorepro_expert_brief_bilingual.pdf)과 [독립 코호트 확장 요약](reports/extension_summary.md). - **고정 결과 재현:** 아래 재현 절의 명령을 사용하십시오. +- **제3자 데이터셋 또는 예측기 추가:** [plug-in contract](contracts/README.md)를 참조하십시오. - **고정 버전 인용:** [CITATION.cff](CITATION.cff)와 [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)를 참조하십시오. 과학적 연구 계약과 범위는 [RESEARCH_SPEC.md](RESEARCH_SPEC.md)를 참조하십시오. @@ -22,10 +23,12 @@ NeoRepro는 공개 MHC-I 펩타이드–HLA 신생항원 예측기를 위한 데 ## 상태 - 최신 문헌 감사: 완료, 결정은 `RESCOPE, then GO` -- 재현한 예측기: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN, DeepHLApan +- 벤치마크 예측기: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN, DeepHLApan. 추가로 일곱 공개 도구의 프로필 전용, 비교 불가 또는 재현 실패 기록을 버전 관리 - TESLA 파일럿: 완료, 학습 데이터 중복 양성 대조군으로 재분류 -- 주요 벤치마크: IMPROVE, 누출 필터링 후 17,475개 레코드, 70명 환자, 3개 코호트 +- 주요 벤치마크: IMPROVE, 누출 필터링 후 17,475개 레코드, 70명 환자, 세 개 코호트 - 주요 IMPROVE 추론: 완료, 고정 도구 예측 52,425개, 누락 레코드 없음 +- 외부 도메인: Zhao 백신 코호트와 별도로 고정한 129개 레코드·9명 환자의 RCC 백신 코호트 +- 재사용 가능한 확장 인터페이스: 기계 검증된 Dataset Card, Predictor Card 및 예측 아티팩트 계약 - 원고: [리소스 중심 버전](paper/manuscript_resource.md), 고정 결과 파일에서 생성, 독립 통계 및 생물학 검토 완료 ## 주요 결과 @@ -48,7 +51,7 @@ make -j4 reproduce-results - **감사 가능한 출력:** [최종 결과표](results/final_results.csv), [그림](results/figures/), [학습 데이터 중복 감사](research/training_overlap_summary_improve.json), [SHA-256 매니페스트](results/manifest.json). - **투고 계획:** [대상 저널 전략](reports/target_venues_2026-08-20.md). -독립 Zhao 2026 백신 코호트 확장은 `make -j4 extension`으로 재현할 수 있습니다. 간결한 증거 요약은 [reports/extension_summary.md](reports/extension_summary.md), 추론 전에 고정한 연구 계약은 [research/extension_protocol.json](research/extension_protocol.json)에 있습니다. 외부 종말점은 펩타이드 펄스 수지상세포 투여 후 백신 접종 뒤 ELISPOT이며, 자연 종양 제시나 임상 효능으로 해석해서는 안 됩니다. +독립 Zhao 2026 백신 코호트 확장은 `make -j4 extension`으로 재현할 수 있습니다. 간결한 증거 요약은 [reports/extension_summary.md](reports/extension_summary.md), 추론 전에 고정한 연구 계약은 [research/extension_protocol.json](research/extension_protocol.json)에 있습니다. 별도로 고정한 RCC 확장은 [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), 세 도메인 탐색적 안정성 출력은 `results/analysis/stability/`에 있습니다. 두 외부 종말점 모두 백신 접종 후 측정이며 자연 종양 제시나 임상 효능으로 해석해서는 안 됩니다. ## 라이선스 diff --git a/README.md b/README.md index b690f31..6e79599 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ This is a benchmark/resource contribution, not a new predictor and not a claim o - **Current manuscript:** [resource-positioned manuscript](paper/manuscript_resource.md). - **Two-minute evidence summary:** [bilingual expert brief](output/pdf/neorepro_expert_brief_bilingual.pdf) and [independent-cohort extension summary](reports/extension_summary.md). - **Reproduce the frozen results:** use the command in the Reproduce section below. +- **Add a third-party dataset or predictor:** follow the [plug-in contract](contracts/README.md). - **Cite a fixed version:** [CITATION.cff](CITATION.cff) and the [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). For the scientific contract and scope, see [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -22,10 +23,12 @@ For the scientific contract and scope, see [RESEARCH_SPEC.md](RESEARCH_SPEC.md). ## Status - Current-literature audit: complete; decision `RESCOPE, then GO` -- Reproduced predictors: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan +- Benchmarked predictors: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan; seven additional public tools have versioned profile-only, non-comparable or failed reproduction records - TESLA pilot: complete; reclassified as a training-overlap-positive control - Primary benchmark: IMPROVE, 17,475 leakage-filtered rows, 70 patients, 3 cohorts - Primary IMPROVE inference: complete; 52,425 fixed-tool predictions with no missing rows +- External domains: Zhao vaccine cohort plus a separately frozen 129-record, 9-patient RCC vaccine cohort +- Reusable extension interface: machine-validated Dataset Card, Predictor Card and prediction-artifact contracts - Manuscript: [resource-positioned version](paper/manuscript_resource.md), generated from frozen result files; independent statistical and biological review complete ## Main result @@ -48,7 +51,7 @@ Independent bootstrap analyses are parallelized by Make. Use `make reproduce-res - **Auditable outputs:** [final result table](results/final_results.csv), [figures](results/figures/), [training-overlap audit](research/training_overlap_summary_improve.json), and [SHA-256 manifest](results/manifest.json). - **Submission planning:** [target-venue strategy](reports/target_venues_2026-08-20.md). -The independent Zhao 2026 vaccine-cohort extension is reproduced with `make -j4 extension`. Its concise evidence summary is in [reports/extension_summary.md](reports/extension_summary.md), with the frozen pre-inference contract in [research/extension_protocol.json](research/extension_protocol.json). The external endpoint is post-vaccination ELISPOT after peptide-pulsed dendritic-cell administration and must not be interpreted as natural tumor presentation or clinical efficacy. +The independent Zhao 2026 vaccine-cohort extension is reproduced with `make -j4 extension`. Its concise evidence summary is in [reports/extension_summary.md](reports/extension_summary.md), with the frozen pre-inference contract in [research/extension_protocol.json](research/extension_protocol.json). The separately frozen RCC extension is documented in [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), and exploratory three-domain stability outputs are under `results/analysis/stability/`. Both external endpoints are post-vaccination assays and must not be interpreted as natural tumor presentation or clinical efficacy. ## License diff --git a/README.zh-CN.md b/README.zh-CN.md index f7c98cc..037b92b 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -15,6 +15,7 @@ NeoRepro 是面向公开 MHC-I 肽–HLA 新抗原预测器的泄漏感知、患 - **当前论文稿：**[资源定位版论文](paper/manuscript_resource.md)。 - **两分钟证据摘要：**[中英双语专家简报](output/pdf/neorepro_expert_brief_bilingual.pdf)和[独立队列扩展摘要](reports/extension_summary.md)。 - **复现冻结结果：**使用下方“复现”部分中的命令。 +- **接入第三方数据集或预测器：**请参阅 [plug-in contract](contracts/README.md)。 - **引用固定版本：**参见 [CITATION.cff](CITATION.cff) 和 [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)。 科学研究契约与范围见 [RESEARCH_SPEC.md](RESEARCH_SPEC.md)。 @@ -22,10 +23,12 @@ NeoRepro 是面向公开 MHC-I 肽–HLA 新抗原预测器的泄漏感知、患 ## 状态 - 最新文献审计：已完成；结论为 `RESCOPE, then GO` -- 已复现预测器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 和 DeepHLApan +- 已纳入基准的预测器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 和 DeepHLApan；另有七个公开工具保留了仅档案、不可比较或复现失败的版本化记录 - TESLA 试点：已完成；重新归类为训练数据重叠阳性对照 - 主要基准：IMPROVE，17,475 条经过泄漏过滤的记录、70 名患者、3 个队列 - 主要 IMPROVE 推理：已完成；52,425 条固定工具预测，无缺失记录 +- 外部领域：Zhao 疫苗队列，以及单独冻结的 RCC 疫苗队列（129 条记录、9 名患者） +- 可复用扩展接口：经机器验证的 Dataset Card、Predictor Card 和预测工件契约 - 论文稿：[资源定位版](paper/manuscript_resource.md)，由冻结结果文件生成；独立统计学与生物学审查均已完成 ## 主要结果 @@ -48,7 +51,7 @@ Make 会并行执行相互独立的 bootstrap 分析。在 CPU 或内存受限 - **可审计输出：**[最终结果表](results/final_results.csv)、[图表](results/figures/)、[训练数据重叠审计](research/training_overlap_summary_improve.json)和 [SHA-256 清单](results/manifest.json)。 - **投稿规划：**[目标期刊策略](reports/target_venues_2026-08-20.md)。 -独立 Zhao 2026 疫苗队列扩展可通过 `make -j4 extension` 复现。简要证据摘要见 [reports/extension_summary.md](reports/extension_summary.md)，推理前冻结的研究契约见 [research/extension_protocol.json](research/extension_protocol.json)。外部终点是在肽脉冲树突状细胞给药后的疫苗接种后 ELISPOT，不应解读为自然肿瘤呈递或临床疗效。 +独立 Zhao 2026 疫苗队列扩展可通过 `make -j4 extension` 复现。简要证据摘要见 [reports/extension_summary.md](reports/extension_summary.md)，推理前冻结的研究契约见 [research/extension_protocol.json](research/extension_protocol.json)。单独冻结的 RCC 扩展记录在 [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json)，三领域探索性稳定性输出位于 `results/analysis/stability/`。两个外部终点均为疫苗接种后检测，不应解读为自然肿瘤呈递或临床疗效。 ## 许可证 diff --git a/README.zh-TW.md b/README.zh-TW.md index 83867e7..aa20581 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -15,6 +15,7 @@ NeoRepro 是一套針對公開 MHC-I 胜肽–HLA 新抗原預測器、考量資 - **目前論文稿：**[資源定位版論文](paper/manuscript_resource.md)。 - **兩分鐘證據摘要：**[中英雙語專家簡報](output/pdf/neorepro_expert_brief_bilingual.pdf)與[獨立隊列延伸摘要](reports/extension_summary.md)。 - **重現凍結結果：**使用下方「重現」章節中的指令。 +- **接入第三方資料集或預測器：**請參閱 [plug-in contract](contracts/README.md)。 - **引用固定版本：**請參閱 [CITATION.cff](CITATION.cff) 與 [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)。 科學研究契約與範圍請見 [RESEARCH_SPEC.md](RESEARCH_SPEC.md)。 @@ -22,10 +23,12 @@ NeoRepro 是一套針對公開 MHC-I 胜肽–HLA 新抗原預測器、考量資 ## 狀態 - 最新文獻稽核：已完成；決策為 `RESCOPE, then GO` -- 已重現的預測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 與 DeepHLApan +- 已納入基準的預測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 與 DeepHLApan；另有七個公開工具保留僅供描述、不可比較或重現失敗的版本化記錄 - TESLA 試驗：已完成；重新歸類為訓練資料重疊陽性對照 - 主要基準：IMPROVE，17,475 筆經資料洩漏過濾的記錄、70 位病人、3 個隊列 - 主要 IMPROVE 推論：已完成；52,425 筆固定工具預測，沒有缺漏記錄 +- 外部領域：Zhao 疫苗隊列，以及另行凍結的 RCC 疫苗隊列（129 筆記錄、9 位病人） +- 可重用延伸介面：經機器驗證的 Dataset Card、Predictor Card 與預測產物契約 - 論文稿：[資源定位版](paper/manuscript_resource.md)，由凍結結果檔案產生；獨立統計與生物學審查皆已完成 ## 主要結果 @@ -48,7 +51,7 @@ Make 會平行執行彼此獨立的 bootstrap 分析。CPU 或記憶體受限時 - **可稽核輸出：**[最終結果表](results/final_results.csv)、[圖表](results/figures/)、[訓練資料重疊稽核](research/training_overlap_summary_improve.json)與 [SHA-256 清單](results/manifest.json)。 - **投稿規劃：**[目標期刊策略](reports/target_venues_2026-08-20.md)。 -獨立 Zhao 2026 疫苗隊列延伸可用 `make -j4 extension` 重現。簡要證據摘要位於 [reports/extension_summary.md](reports/extension_summary.md)，推論前凍結的研究契約位於 [research/extension_protocol.json](research/extension_protocol.json)。外部終點是以胜肽脈衝樹突細胞施打後的疫苗接種後 ELISPOT，不應解讀為自然腫瘤呈遞或臨床療效。 +獨立 Zhao 2026 疫苗隊列延伸可用 `make -j4 extension` 重現。簡要證據摘要位於 [reports/extension_summary.md](reports/extension_summary.md)，推論前凍結的研究契約位於 [research/extension_protocol.json](research/extension_protocol.json)。另行凍結的 RCC 延伸記錄於 [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json)，三領域探索性穩定性輸出位於 `results/analysis/stability/`。兩個外部終點均為疫苗接種後檢測，不應解讀為自然腫瘤呈遞或臨床療效。 ## 授權條款 diff --git a/contracts/README.md b/contracts/README.md new file mode 100644 index 0000000..2e67bad --- /dev/null +++ b/contracts/README.md @@ -0,0 +1,13 @@ +# NeoRepro plug-in contract + +Third parties can validate cards and artifacts, then evaluate in one command: + +```bash +neorepro dataset validate contracts/dataset-card.example.json +neorepro predictor validate contracts/predictor-card.example.json +neorepro artifact predictions.csv --benchmark benchmark.csv +neorepro evaluate benchmark.csv predictions.csv --output results/evaluation.json --report reports/evaluation.md +neorepro overlap-audit predictions.csv +``` + +Every artifact must contain one row for every benchmark `record_id`. Failed or unsupported predictions remain explicit rows and are counted as missing; they are never imputed. `score_direction` is mandatory and evaluation orients lower-is-better scores before metrics. Unknown training overlap is reported as unknown, not as independence. diff --git a/contracts/dataset-card.example.json b/contracts/dataset-card.example.json new file mode 100644 index 0000000..97e37da --- /dev/null +++ b/contracts/dataset-card.example.json @@ -0,0 +1,10 @@ +{ + "schema_version": "dataset-card-1", + "dataset_id": "synthetic-demo", + "version": "1.0.0", + "records_path": "benchmark.csv", + "label_column": "label", + "patient_id_column": "patient_id", + "score_tasks": ["immunogenicity"], + "source": {"url": "https://example.org", "accessed": "2026-08-20", "sha256": "unknown", "license": "CC-BY-4.0"} +} diff --git a/contracts/prediction-artifact.schema.json b/contracts/prediction-artifact.schema.json new file mode 100644 index 0000000..b757730 --- /dev/null +++ b/contracts/prediction-artifact.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "NeoRepro Prediction Artifact", + "type": "object", + "required": ["record_id", "predictor", "predictor_version", "task", "score", "score_direction", "status"], + "properties": { + "record_id": {"type": "string"}, "predictor": {"type": "string"}, "predictor_version": {"type": "string"}, + "task": {"type": "string"}, "score": {"type": ["number", "string", "null"]}, + "score_direction": {"enum": ["higher", "lower"]}, "status": {"enum": ["predicted", "unsupported", "failed", "invalid"]} + } +} diff --git a/contracts/predictor-card.example.json b/contracts/predictor-card.example.json new file mode 100644 index 0000000..79f1671 --- /dev/null +++ b/contracts/predictor-card.example.json @@ -0,0 +1,12 @@ +{ + "schema_version": "predictor-card-1", + "predictor_id": "synthetic-demo", + "version": "1.0.0", + "task": "immunogenicity", + "score_direction": "higher", + "adapter": "predictors/example/adapter.py", + "license": "MIT", + "input": ["record_id", "peptide", "hla"], + "output": "contracts/prediction-artifact.schema.json", + "training_overlap": "unknown" +} diff --git a/contracts/synthetic/README.md b/contracts/synthetic/README.md new file mode 100644 index 0000000..83bb108 --- /dev/null +++ b/contracts/synthetic/README.md @@ -0,0 +1,3 @@ +# Synthetic plug-in example + +This four-row fixture demonstrates explicit unsupported output, common-support evaluation, and patient-level ranking without using biological data. diff --git a/contracts/synthetic/benchmark.csv b/contracts/synthetic/benchmark.csv new file mode 100644 index 0000000..b776520 --- /dev/null +++ b/contracts/synthetic/benchmark.csv @@ -0,0 +1,5 @@ +record_id,patient_id,peptide,hla,label +s1,P1,SIINFEKL,HLA-A*02:01,1 +s2,P1,NLVPMVATV,HLA-A*02:01,0 +s3,P2,GLCTLVAML,HLA-A*02:01,1 +s4,P2,LLD, HLA-A*02:01,0 diff --git a/contracts/synthetic/predictions.csv b/contracts/synthetic/predictions.csv new file mode 100644 index 0000000..1dd4095 --- /dev/null +++ b/contracts/synthetic/predictions.csv @@ -0,0 +1,5 @@ +record_id,predictor,predictor_version,task,score,score_direction,status +s1,synthetic,1.0,immunogenicity,0.9,higher,predicted +s2,synthetic,1.0,immunogenicity,0.1,higher,predicted +s3,synthetic,1.0,immunogenicity,0.8,higher,predicted +s4,synthetic,1.0,immunogenicity,,higher,unsupported diff --git a/data/predictor_registry.csv b/data/predictor_registry.csv index 94f9ab9..e25085f 100644 --- a/data/predictor_registry.csv +++ b/data/predictor_registry.csv @@ -4,3 +4,10 @@ BigMHC,v1.0,9d84a3b4da77c9253ac90ff8cb629274003b90fd,BigMHC-Academic-License,yes PRIME,2.0,ec1aa020089d62e9193ad377ddda9c93eed7f5b1,LICR-academic-noncommercial,yes,no,yes,yes,yes,complete,8.50,yes,none,no,no,no,macOS-15.3-arm64,no,no,yes,reproduced,predictors/prime/attempts/,"17,475/17,475 IMPROVE predictions plus the 520-row leakage-positive fixture; requires MixMHCpred v2.2 at f64bb4548082768c70a1cfb5a4442d5e6ea04591; both C++ binaries recompiled for ARM64; 146-row official fixture matched exactly; upstream artifacts are not redistributed." DeepImmuno-CNN,1.0,df42ac5b6bddfe531268335e2dcb496559cd488b,MIT,no,yes,yes,yes,yes,complete,unknown,yes,none,no,no,no,macOS-15.3-arm64,no,no,yes,reproduced,predictors/deepimmuno/,"Published checkpoint loaded under isolated TensorFlow 2.15.1 environment; exact HLA only, no upstream fuzzy rescue; 9–10mer contract; public 9,055-row training table audited for exact overlap." DeepHLApan,1.1.1,ac1f4bebc095271504dfc2d2a93888df3be94e83,GPL-2.0,no,yes,yes,yes,yes,complete,unknown,yes,legacy_Keras_GRU_loader,no,no,no,macOS-15.3-arm64,no,no,yes,reproduced,predictors/deephlapan/,"All five published binding and immunogenicity models load through a reset_after=False compatibility shim and are averaged; exact HLA only. Official row-level training manifest unavailable, so overlap is unknown." +MHCnuggets,2.4.0,b666fea3a54a1d357efba4ea4d8550ce5dd50aba,unclear,unknown,unknown,yes,yes,yes,complete,7.53,no,none,no,no,no,macOS-15.3-arm64,optional,no,yes,pending,predictors/mhcnuggets/attempts/,"Production BA weights are vendored in the pinned source; official test peptide fixture ran through the upstream API and strict adapter (3/3). Lower IC50 is better. The repository's separate saves/test model has different expected values, so exact test-model comparison is not claimed. Binding-track comparability and training-overlap audit remain pending." +NeoGuider,unknown,f95a5713453910902d657f1d148568b93c81026f,AGPL-3.0,unknown,yes,yes,no,no,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,unknown,yes,unknown,attempted_unreproducible,predictors/neoguider/attempts/,"Repository lacks pyproject.toml/setup.py; documented route is shell/conda. No benchmark entry." +NeoFox,1.2.3,dd894a67ad02a4377ed3e4b1e44bd576360c98cd,GPL-3.0,no,yes,yes,yes,yes,none,unknown,no,none,no,no,unknown,macOS-15.3-arm64,no,optional,yes,pending,predictors/neofox/attempts/,"Editable install and import smoke test passed; annotation output is not peptide-HLA score comparable. Profile-only pending official fixture and service audit." +pVACtools,7.1.2,2ce871be5ae56af141fe723d60d3841125874a74,BSD-3-Clause-Clear,no,yes,yes,yes,yes,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,no,optional,yes,excluded_noncomparable,predictors/pvactools/attempts/,"Editable install and pvacseq --help smoke test passed; end-to-end orchestration, not a single comparable score model." +Seq2Neo,2.1,f342aa4ffa9012fb5509914333b9401a1eb286c4,AFL-3.0,unknown,yes,yes,no,no,none,unknown,no,dependency_conflict:no_tensorflow_2.3_cp311,no,no,unknown,macOS-15.3-arm64,unknown,yes,unknown,attempted_unreproducible,predictors/seq2neo/attempts/,"Install failed under Python 3.11 because tensorflow==2.3.0 has no cp311 wheel; legacy route remains follow-up. Profile-only." +Vaxrank,unknown,5c71edccda920fafaab5986837d3dd87c8ce8911,Apache-2.0,no,yes,yes,yes,yes,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,no,optional,yes,excluded_noncomparable,predictors/vaxrank/attempts/,"Editable install and help smoke test passed; vaccine-construct ranking is not peptide-HLA comparable." +mhcmatch,0.25.0,956bdb613fdb1e128cbd7d59fabb9b47ef1aaae9,GPL-3.0,no,yes,yes,yes,yes,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,no,no,unknown,pending,predictors/mhcmatch/attempts/,"Editable install, --help, and local decompose smoke tests passed. Binder score timed out after 180s while triggering calibration/reference bootstrap; preserved as a resource/data dependency result. Exploratory only pending method paper and training provenance." diff --git a/data/processed/rcc_vaccine_benchmark.csv b/data/processed/rcc_vaccine_benchmark.csv new file mode 100644 index 0000000..54540f3 --- /dev/null +++ b/data/processed/rcc_vaccine_benchmark.csv @@ -0,0 +1,130 @@ +record_id,patient_id,study_id,hla,mhc_class,peptide,peptide_length,immunogenicity,label,assay_type,clinical_context,source_doi,source_url,source_checksum,source_row,source_pvalue,evidence_level +rcc-vaccine-8c2e1ec6d8f675cf,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,2,5.6006335776299125E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6a6ffb8b81ec9208,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,KEKIREYEQAL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,3,4.4603377394004801E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4adeedbe0e35cf09,RCC-101,RCC_PCV_VACCINE_2025,HLA-C*06:02;HLA-A*66:01,I,VRNSRTSGY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,4,8.8070660481353924E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e06799bdf8de8af5,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,5,3.7959293105110106E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-969998ab9d8b4dc1,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,SVASHGGHQPY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,6,0.89810982568718123,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a5952e9e71bd27bd,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,GEILLPRDV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,7,0.26875586775158017,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6fa3e9ad450e030e,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,8,0.73956419203215784,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8a9041cc55b08e38,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,TETDSDPMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,9,4.3972842081206977E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e11c81feedc5751e,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,LFRGLQSR,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,10,1.1492768296259299E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fefffdcccca8649c,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,REGTGAVGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,11,5.6966328761325709E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-940b62df9c3078c4,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,ITLPVYSER,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,12,2.1248320025947062E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-43742d14e082c2e6,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,RAAASSRRPS,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,13,5.828028176445943E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-769c6b305827c570,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,FPSETNLSKEV,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,14,0.10474802857947568,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e74f183589095f61,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,NAFVEEVM,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,15,0.17473444899797558,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-15ab70cb934aad8f,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,FHFVIYVPI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,16,5.0774964296645174E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c8a8ce5ab15f9d4b,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,LALPLQRTW,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,17,0.92524545777236522,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f05602883865f4bf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,LPFDLKNPF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,18,0.80801446661256293,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b92105a12fb6f028,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,AHYSTDTAL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,19,0.47154937914914336,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a004075aed453bcf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:01,I,LPSMQEDL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,20,0.3095552136678249,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70eb710f90e2973b,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,EPAQFPEPNSF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,21,3.6050424739501194E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c81528933be4a147,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,FQESYFESY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,22,9.2338784047006368E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-2a2f0b8f959aa3f6,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,HPPPHIQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,23,0.21566601684775977,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-74e8a99b7ef86c61,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LLYNKMIL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,24,1.2335170096117602E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88bea2527dd72a1d,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,GSPAITHRNL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,25,2.3339160452827509E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ff6bbb8d69468a47,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,SRNDVSRL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,26,2.5026099459392787E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8ea6c4f21d2c81ee,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ILLGEDDSVQI,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,27,0.18350341907227408,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2f6110d808ac0da,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,IVGPLPPPL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,28,0.2254033307585166,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f43e93564087f160,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,FRPDSKSSAL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,29,3.5098718645984656E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fc1276f53c3f45e1,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,KKGRPGQTGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,30,0.19930188719405426,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3e6b1c1d8de0ce71,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,VAFDFTKVE,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,31,0.47311639825056773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4255bdded9e984bd,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,SRGAAGWSL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,32,0.57898632350489265,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-31c3a5a1c962db43,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LPRPKIARL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,33,8.3377373835244337E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d3d4046d155047b6,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ALSEQIQMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,34,1.4246046683842653E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5e4096b57e93c92e,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,LLKLLHHRY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,35,2.0118966840664707E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e8d6aeb51705642a,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,ARWGETVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,36,1.8306664737796626E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-26dc24e186221fe2,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,GTKPLYVVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,37,5.0221254228049662E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-364822c022710958,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,ILKRRTNM,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,38,4.7676050558719381E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9c477d13a3359f5e,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,39,7.0908120808195439E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-487a8e43d12c17ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,EILERRFL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,40,0.13919156951235334,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-539b50b223ea0ee1,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,41,2.3027081004049618E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-618d6aa7d8232657,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,42,8.1766449056166948E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-71f5d2ab19a389e8,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,STATLPFHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,43,1.846734671054058E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-409a0661a319bbb1,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,44,6.6867747669650493E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9f431d040990f0f7,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,YSNWPDKLY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,45,8.0583222939814639E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7c1009a0bb0524b3,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,AFFDGKTL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,46,4.8824113766337794E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-42e869aba90dc4bd,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ISHISPQRGFH,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,47,1.7166906883565791E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a1a88d1c11b4fcdd,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,GGGAEKVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,48,0.42264973081037416,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d42b9fb3a44366ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,49,1.021179201718677E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ce806d8b1a4132b2,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,50,2.3380484295577471E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ea51c18cc4e16995,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,51,1.7222634968187069E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9bce007ba3a4aff0,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,STGLRVIK,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,52,1.0052468182969208E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa3019acbdddb7dc,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,YGCPHLQQ,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,53,0.16109342700158688,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1bcdaed331509a08,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,54,5.9493457814588473E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4a1853322eea7a14,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,MRILHNKGI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,55,2.131276725678492E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-cde49959d75e6ec2,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FMNSSTSL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,56,1.773461347691256E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7f8e1f8e02332663,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ESQKEMKLLLY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,57,0.67173725533059248,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8aa4d97d214f765,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,MSIPGMEEQL,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,58,6.0793325180986964E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5993797f7a6c7057,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,RLRDDVMRLSR,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,59,1.400860297501039E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-eacb2f310488ca9d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,VEAFDADMLKL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,60,1.214858610163266E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70f05fcfa038224f,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,HSPKYPSPYR,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,61,0.67085616631628397,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d15c4e62bb19311,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,PPKSLFQL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,62,4.6693166687862796E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a6b2180415ff1b56,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,KPEIAHVTSA,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,63,4.2382661368441126E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e52f3609d694b4ba,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KLDTGFKILDR,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,64,0.27589910187209943,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-763f29ff7ac4c65d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,METIEKIMGDL,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,65,0.76746069478425916,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c331f65899396b9b,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,YVISRTSPV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,66,8.0195333258099456E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-09c9b699d977f421,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,KLWLLRPLR,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,67,7.1646552252994013E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e944d5a782a7d074,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,LDAPDFQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,68,0.51383607553050725,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-75f86cf28dca60e2,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,IINEASQVKPK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,69,7.2477924894824214E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5b2ef450eaf2248a,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,FAFYGAHGAQF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,70,6.5233637469130547E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c50e7f0e1b3b69a5,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KAKPVIFLGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,71,7.0798177776713822E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ae70f68909b113e8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,DEITATYML,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,72,7.0303170829206542E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-efdf21635c142ab8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,SPYHYQRV,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,73,5.6759276880420054E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-817b1ac6328997a7,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KIMKKAIERGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,74,1.6379409412884873E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-feb66d3c66c8a4de,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,GLWELPVEGRA,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,75,0.19170962313452389,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7946e9a10fe114cf,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LDMKGEIAEAY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,76,4.8810268788658163E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f3e32e857dd09d71,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,AENNDLLWMNY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,77,0.43819304962341288,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-240bf5b232815c6a,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VIEPHDILV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,78,1.6539295298740098E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-40c01a17a3ed4a2e,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,KLYEAEDDFKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,79,5.025231714396812E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7b823cc728ea1450,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,RILDAREHSY,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,80,0.23019964108049898,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0c7eb5e3295f0ad2,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,REVRLQNKTSY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,81,3.7355310355967439E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88aaf2311ce63196,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,ASDGLSERPL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,82,8.1513907431533788E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-bcae618cfad77c1f,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,PDMVVSHLFW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,83,0.56115088124008583,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-11f61d47947266cc,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KLQQPDHASW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,84,2.2184135540696064E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f4a4db2362e7a54e,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,86,3.774955135062371E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1eceb5b6a969d721,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,SRKGSIKGLVY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,87,7.417990022744847E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-55106745d24cfbf4,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,SLNFKNPEA,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,88,1.3963573524385535E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a808947689fbe42c,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,FRFTPPNPEA,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,89,0.11808289631180313,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-590f5ea08f24e2cf,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,PRHAAGGLHF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,90,5.9124927719229947E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b984dc3be5c1fa17,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,91,8.2441737161747519E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6ff6e758ac279425,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YAVPGPIVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,92,6.3803433637722395E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa0d4e0503dacba4,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,WRELNDLGSK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,93,0.43261607750026987,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d44829f31cf5558,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRPETFNHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,94,2.9913865953409766E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4f7d9f9f22d48440,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRLSHFEYVKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,95,0.7467841772335857,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-60d1ba1b8ae350fd,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,96,1.281550265552316E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7ec85836cb09ee30,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:06,I,HQYVPKDQRL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,97,1.4725369350184756E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4b7fdcc54f65b98b,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,98,9.0220118727937296E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d02de725ecc685a5,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARTFDKKRF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,99,5.8042374560036136E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e1d984ebf245728f,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,LRSILPSTGI,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,100,3.5960591985765922E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a9a3c2bc98a74746,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,101,4.5821799894541555E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-da610226e338a8de,RCC-109,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YLMDISGKV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,102,2.448656045672399E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ca0a8a8bb4e2d094,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,AAGSASSSKV,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,103,4.0408082142463435E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c45585f253341070,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,104,4.5974381758390748E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2761351ccb757b7,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,105,0.29539930932276304,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b1ad936ccc2f249a,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,106,5.9503855187285459E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-02261377bb800592,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,107,4.7874694511645952E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-834aff6d2a7dc363,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,108,2.2477796563320426E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0036b8c45a4cd20d,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,109,0.37853076038781203,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-08447808dd0300dd,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,110,6.4833740564642257E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-34c3e7e7a2dcd8b6,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,111,6.7901063097408093E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1ae968c5ae01f49b,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,112,2.1954978277654386E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-29f7c92d094c1ced,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,EEVLEIAASGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,113,2.2356452288465851E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0aaef5904890371c,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,114,2.8827717670021147E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-27b47d43dab2b353,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VTDTFSDV,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,115,0.24291648886267675,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-46b486b9d199e93a,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,ARGPMPVHTT,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,116,6.2291361519685288E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8a9c44bcf53568f,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KVLKLIPEK,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,117,4.7559190547222883E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7425139a6230adc0,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,118,1.1997646355098002E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e39c49c350f42009,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,RPSQPSRERL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,119,4.6026306166334421E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-73717d4619b66faf,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,MREIREVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,120,1,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f130cbb5ea1ca0fa,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,HSLEMMNSM,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,121,1.9687903060187317E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3f16099c911d2791,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,122,0.28825566505998773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-07bc8054e1edd65a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,VPNHLPPAL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,123,2.378495810276827E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-99bd603a759d6b22,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,RYSIHSEL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,124,3.0348245596030862E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ad726d8ef91bd0e6,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,SRGDGEHF,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,125,6.6703592263578429E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-972e1f28fe4cb96a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEQNATKSW,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,126,1.6695522924906413E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e0d628668c1eccf2,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*25:01,I,DIEDGEADHI,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,127,9.8999275264303174E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3d165d772065bb3a,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,YAFSPPQGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,128,3.3849045480144387E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9ef49682993d2ea1,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,PPAPQPQRL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,129,0.65512239646493831,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6c01a27a6a575007,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,ARWHRAQDL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,130,1.266837038557146E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-12b152d35907e280,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,KPPKAYSVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,131,0.19485075298185836,individual_peptide_stimulation_with_matched_no_stimulation_control diff --git a/data/processed/rcc_vaccine_benchmark_full.csv b/data/processed/rcc_vaccine_benchmark_full.csv new file mode 100644 index 0000000..54540f3 --- /dev/null +++ b/data/processed/rcc_vaccine_benchmark_full.csv @@ -0,0 +1,130 @@ +record_id,patient_id,study_id,hla,mhc_class,peptide,peptide_length,immunogenicity,label,assay_type,clinical_context,source_doi,source_url,source_checksum,source_row,source_pvalue,evidence_level +rcc-vaccine-8c2e1ec6d8f675cf,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,2,5.6006335776299125E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6a6ffb8b81ec9208,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,KEKIREYEQAL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,3,4.4603377394004801E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4adeedbe0e35cf09,RCC-101,RCC_PCV_VACCINE_2025,HLA-C*06:02;HLA-A*66:01,I,VRNSRTSGY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,4,8.8070660481353924E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e06799bdf8de8af5,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,5,3.7959293105110106E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-969998ab9d8b4dc1,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,SVASHGGHQPY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,6,0.89810982568718123,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a5952e9e71bd27bd,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,GEILLPRDV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,7,0.26875586775158017,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6fa3e9ad450e030e,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,8,0.73956419203215784,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8a9041cc55b08e38,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,TETDSDPMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,9,4.3972842081206977E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e11c81feedc5751e,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,LFRGLQSR,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,10,1.1492768296259299E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fefffdcccca8649c,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,REGTGAVGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,11,5.6966328761325709E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-940b62df9c3078c4,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,ITLPVYSER,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,12,2.1248320025947062E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-43742d14e082c2e6,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,RAAASSRRPS,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,13,5.828028176445943E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-769c6b305827c570,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,FPSETNLSKEV,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,14,0.10474802857947568,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e74f183589095f61,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,NAFVEEVM,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,15,0.17473444899797558,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-15ab70cb934aad8f,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,FHFVIYVPI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,16,5.0774964296645174E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c8a8ce5ab15f9d4b,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,LALPLQRTW,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,17,0.92524545777236522,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f05602883865f4bf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,LPFDLKNPF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,18,0.80801446661256293,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b92105a12fb6f028,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,AHYSTDTAL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,19,0.47154937914914336,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a004075aed453bcf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:01,I,LPSMQEDL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,20,0.3095552136678249,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70eb710f90e2973b,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,EPAQFPEPNSF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,21,3.6050424739501194E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c81528933be4a147,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,FQESYFESY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,22,9.2338784047006368E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-2a2f0b8f959aa3f6,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,HPPPHIQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,23,0.21566601684775977,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-74e8a99b7ef86c61,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LLYNKMIL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,24,1.2335170096117602E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88bea2527dd72a1d,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,GSPAITHRNL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,25,2.3339160452827509E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ff6bbb8d69468a47,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,SRNDVSRL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,26,2.5026099459392787E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8ea6c4f21d2c81ee,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ILLGEDDSVQI,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,27,0.18350341907227408,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2f6110d808ac0da,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,IVGPLPPPL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,28,0.2254033307585166,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f43e93564087f160,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,FRPDSKSSAL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,29,3.5098718645984656E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fc1276f53c3f45e1,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,KKGRPGQTGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,30,0.19930188719405426,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3e6b1c1d8de0ce71,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,VAFDFTKVE,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,31,0.47311639825056773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4255bdded9e984bd,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,SRGAAGWSL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,32,0.57898632350489265,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-31c3a5a1c962db43,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LPRPKIARL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,33,8.3377373835244337E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d3d4046d155047b6,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ALSEQIQMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,34,1.4246046683842653E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5e4096b57e93c92e,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,LLKLLHHRY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,35,2.0118966840664707E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e8d6aeb51705642a,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,ARWGETVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,36,1.8306664737796626E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-26dc24e186221fe2,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,GTKPLYVVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,37,5.0221254228049662E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-364822c022710958,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,ILKRRTNM,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,38,4.7676050558719381E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9c477d13a3359f5e,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,39,7.0908120808195439E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-487a8e43d12c17ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,EILERRFL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,40,0.13919156951235334,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-539b50b223ea0ee1,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,41,2.3027081004049618E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-618d6aa7d8232657,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,42,8.1766449056166948E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-71f5d2ab19a389e8,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,STATLPFHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,43,1.846734671054058E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-409a0661a319bbb1,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,44,6.6867747669650493E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9f431d040990f0f7,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,YSNWPDKLY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,45,8.0583222939814639E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7c1009a0bb0524b3,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,AFFDGKTL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,46,4.8824113766337794E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-42e869aba90dc4bd,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ISHISPQRGFH,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,47,1.7166906883565791E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a1a88d1c11b4fcdd,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,GGGAEKVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,48,0.42264973081037416,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d42b9fb3a44366ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,49,1.021179201718677E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ce806d8b1a4132b2,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,50,2.3380484295577471E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ea51c18cc4e16995,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,51,1.7222634968187069E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9bce007ba3a4aff0,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,STGLRVIK,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,52,1.0052468182969208E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa3019acbdddb7dc,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,YGCPHLQQ,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,53,0.16109342700158688,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1bcdaed331509a08,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,54,5.9493457814588473E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4a1853322eea7a14,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,MRILHNKGI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,55,2.131276725678492E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-cde49959d75e6ec2,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FMNSSTSL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,56,1.773461347691256E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7f8e1f8e02332663,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ESQKEMKLLLY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,57,0.67173725533059248,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8aa4d97d214f765,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,MSIPGMEEQL,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,58,6.0793325180986964E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5993797f7a6c7057,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,RLRDDVMRLSR,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,59,1.400860297501039E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-eacb2f310488ca9d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,VEAFDADMLKL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,60,1.214858610163266E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70f05fcfa038224f,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,HSPKYPSPYR,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,61,0.67085616631628397,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d15c4e62bb19311,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,PPKSLFQL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,62,4.6693166687862796E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a6b2180415ff1b56,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,KPEIAHVTSA,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,63,4.2382661368441126E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e52f3609d694b4ba,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KLDTGFKILDR,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,64,0.27589910187209943,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-763f29ff7ac4c65d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,METIEKIMGDL,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,65,0.76746069478425916,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c331f65899396b9b,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,YVISRTSPV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,66,8.0195333258099456E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-09c9b699d977f421,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,KLWLLRPLR,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,67,7.1646552252994013E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e944d5a782a7d074,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,LDAPDFQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,68,0.51383607553050725,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-75f86cf28dca60e2,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,IINEASQVKPK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,69,7.2477924894824214E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5b2ef450eaf2248a,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,FAFYGAHGAQF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,70,6.5233637469130547E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c50e7f0e1b3b69a5,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KAKPVIFLGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,71,7.0798177776713822E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ae70f68909b113e8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,DEITATYML,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,72,7.0303170829206542E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-efdf21635c142ab8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,SPYHYQRV,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,73,5.6759276880420054E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-817b1ac6328997a7,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KIMKKAIERGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,74,1.6379409412884873E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-feb66d3c66c8a4de,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,GLWELPVEGRA,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,75,0.19170962313452389,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7946e9a10fe114cf,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LDMKGEIAEAY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,76,4.8810268788658163E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f3e32e857dd09d71,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,AENNDLLWMNY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,77,0.43819304962341288,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-240bf5b232815c6a,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VIEPHDILV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,78,1.6539295298740098E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-40c01a17a3ed4a2e,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,KLYEAEDDFKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,79,5.025231714396812E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7b823cc728ea1450,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,RILDAREHSY,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,80,0.23019964108049898,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0c7eb5e3295f0ad2,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,REVRLQNKTSY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,81,3.7355310355967439E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88aaf2311ce63196,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,ASDGLSERPL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,82,8.1513907431533788E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-bcae618cfad77c1f,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,PDMVVSHLFW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,83,0.56115088124008583,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-11f61d47947266cc,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KLQQPDHASW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,84,2.2184135540696064E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f4a4db2362e7a54e,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,86,3.774955135062371E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1eceb5b6a969d721,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,SRKGSIKGLVY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,87,7.417990022744847E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-55106745d24cfbf4,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,SLNFKNPEA,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,88,1.3963573524385535E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a808947689fbe42c,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,FRFTPPNPEA,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,89,0.11808289631180313,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-590f5ea08f24e2cf,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,PRHAAGGLHF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,90,5.9124927719229947E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b984dc3be5c1fa17,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,91,8.2441737161747519E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6ff6e758ac279425,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YAVPGPIVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,92,6.3803433637722395E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa0d4e0503dacba4,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,WRELNDLGSK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,93,0.43261607750026987,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d44829f31cf5558,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRPETFNHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,94,2.9913865953409766E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4f7d9f9f22d48440,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRLSHFEYVKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,95,0.7467841772335857,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-60d1ba1b8ae350fd,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,96,1.281550265552316E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7ec85836cb09ee30,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:06,I,HQYVPKDQRL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,97,1.4725369350184756E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4b7fdcc54f65b98b,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,98,9.0220118727937296E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d02de725ecc685a5,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARTFDKKRF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,99,5.8042374560036136E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e1d984ebf245728f,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,LRSILPSTGI,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,100,3.5960591985765922E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a9a3c2bc98a74746,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,101,4.5821799894541555E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-da610226e338a8de,RCC-109,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YLMDISGKV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,102,2.448656045672399E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ca0a8a8bb4e2d094,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,AAGSASSSKV,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,103,4.0408082142463435E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c45585f253341070,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,104,4.5974381758390748E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2761351ccb757b7,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,105,0.29539930932276304,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b1ad936ccc2f249a,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,106,5.9503855187285459E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-02261377bb800592,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,107,4.7874694511645952E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-834aff6d2a7dc363,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,108,2.2477796563320426E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0036b8c45a4cd20d,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,109,0.37853076038781203,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-08447808dd0300dd,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,110,6.4833740564642257E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-34c3e7e7a2dcd8b6,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,111,6.7901063097408093E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1ae968c5ae01f49b,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,112,2.1954978277654386E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-29f7c92d094c1ced,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,EEVLEIAASGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,113,2.2356452288465851E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0aaef5904890371c,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,114,2.8827717670021147E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-27b47d43dab2b353,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VTDTFSDV,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,115,0.24291648886267675,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-46b486b9d199e93a,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,ARGPMPVHTT,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,116,6.2291361519685288E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8a9c44bcf53568f,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KVLKLIPEK,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,117,4.7559190547222883E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7425139a6230adc0,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,118,1.1997646355098002E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e39c49c350f42009,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,RPSQPSRERL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,119,4.6026306166334421E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-73717d4619b66faf,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,MREIREVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,120,1,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f130cbb5ea1ca0fa,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,HSLEMMNSM,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,121,1.9687903060187317E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3f16099c911d2791,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,122,0.28825566505998773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-07bc8054e1edd65a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,VPNHLPPAL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,123,2.378495810276827E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-99bd603a759d6b22,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,RYSIHSEL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,124,3.0348245596030862E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ad726d8ef91bd0e6,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,SRGDGEHF,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,125,6.6703592263578429E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-972e1f28fe4cb96a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEQNATKSW,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,126,1.6695522924906413E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e0d628668c1eccf2,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*25:01,I,DIEDGEADHI,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,127,9.8999275264303174E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3d165d772065bb3a,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,YAFSPPQGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,128,3.3849045480144387E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9ef49682993d2ea1,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,PPAPQPQRL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,129,0.65512239646493831,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6c01a27a6a575007,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,ARWHRAQDL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,130,1.266837038557146E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-12b152d35907e280,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,KPPKAYSVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,131,0.19485075298185836,individual_peptide_stimulation_with_matched_no_stimulation_control diff --git a/data/rcc_vaccine_summary.json b/data/rcc_vaccine_summary.json new file mode 100644 index 0000000..8d71fa5 --- /dev/null +++ b/data/rcc_vaccine_summary.json @@ -0,0 +1,9 @@ +{ + "rows": 129, + "patients": 9, + "positives": 75, + "negatives": 54, + "member_sha256": "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1", + "endpoint": "post-vaccine individual-peptide IFNG ELISpot", + "hla_semantics": "predicted best short-epitope binding allele" +} diff --git a/i18n/readme_state.json b/i18n/readme_state.json index 7f891d3..7060526 100644 --- a/i18n/readme_state.json +++ b/i18n/readme_state.json @@ -3,89 +3,89 @@ "de": { "file": "README.de.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "es": { "file": "README.es.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "fr": { "file": "README.fr.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "ja": { "file": "README.ja.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "ko": { "file": "README.ko.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "zh-CN": { "file": "README.zh-CN.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "zh-TW": { "file": "README.zh-TW.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "418af0c8b42d6606208a2d5ef296b960882c4264f3e92795b5e326f2ea13a4d5", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } } }, "source": "README.md", - "source_sha256": "bed740fcc5071ac2386f37de6c67de7633a82ea72ade3cd40fcc211e888240d7", + "source_sha256": "e2a1c7940616892f555199132a46c827471183d54bd0c3703a24273897a2ae1f", "version": 1 } diff --git a/paper/manuscript_resource.md b/paper/manuscript_resource.md index 80463fc..78a16a9 100644 --- a/paper/manuscript_resource.md +++ b/paper/manuscript_resource.md @@ -10,11 +10,11 @@ Public availability of a neoantigen predictor does not ensure that its software, ### Methods -We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and an independent personalized-vaccine ELISPOT cohort; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. +We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and two endpoint-distinct personalized-vaccine ELISPOT cohorts; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. ### Results -All five pinned predictors produced outputs within their declared input support. The initial 520-row TESLA fixture was entirely training-overlapped and was retained only as a leakage-positive reproduction test. After excluding 45 exact PRIME2 overlaps from 17,520 IMPROVE records, 17,475 records from 70 patients remained. On common support, PRIME achieved AUROC 0.597 and mean pMHC-pair Recall@20 0.260 among 60 positive-bearing patients, versus 0.546 and 0.146 for BigMHC. Transparent peptide baselines outperformed HLA-only baselines under both patient- and study-held-out fitting, while adding HLA to peptide features did not consistently improve over peptide features alone. A frozen extension evaluated five models on 2,315 overlap-filtered vaccine peptides with a distinct post-vaccination ELISPOT endpoint. Support-matched random ranking showed that high marginal Top-K values did not necessarily imply useful ranking signal. +All five pinned predictors produced outputs within their declared input support. The initial 520-row TESLA fixture was entirely training-overlapped and was retained only as a leakage-positive reproduction test. After excluding 45 exact PRIME2 overlaps from 17,520 IMPROVE records, 17,475 records from 70 patients remained. On common support, PRIME achieved AUROC 0.597 and mean pMHC-pair Recall@20 0.260 among 60 positive-bearing patients, versus 0.546 and 0.146 for BigMHC. Transparent peptide baselines outperformed HLA-only baselines under both patient- and study-held-out fitting, while adding HLA to peptide features did not consistently improve over peptide features alone. A frozen extension evaluated five models on 2,315 overlap-filtered vaccine peptides with a distinct post-vaccination ELISPOT endpoint. A second endpoint-distinct vaccine cohort contributed 129 individually assayed short peptides from 9 patients. Support-matched and cross-domain analyses showed that high marginal Top-K values did not necessarily imply stable or useful ranking signal. ### Conclusions @@ -32,7 +32,7 @@ NeoRepro asks a narrower question than developing a new predictor: what evidence ### Study design and reproducibility contract -The core protocol was frozen in `RESEARCH_SPEC.md`; the external extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json` before extension inference. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. +The core protocol was frozen in `RESEARCH_SPEC.md`; the Zhao extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json`, and the RCC cohort in `research/extension_protocol_rcc_v1.json`, before their respective inference runs. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. ### Dataset construction and provenance @@ -42,6 +42,8 @@ For the independent extension, we extracted checksum-pinned `Table1.xlsx` from t The administered peptides had been selected by the source workflow using earlier NetMHC/NetMHCpan binding predictions and an IC50 threshold below 500 nM [@zhao2026vaccine]. Consequently, this cohort evaluates reranking within a predictor-enriched candidate set rather than unselected tumor mutations. Reported peptide–HLA restrictions were computational assignments used by the source analysis and were not individually established by HLA-blocking experiments. +For the endpoint-distinct RCC extension, we checksum-pinned Supplementary Table 2 from Braun and colleagues and retained 129 short-peptide rows from nine vaccinated patients after excluding one row without a usable short peptide/HLA assignment [@braun2025rcc]. Every retained row contained three peptide-stimulation and three matched no-stimulation replicates. Positivity followed the source individual-peptide p-value rule. The HLA field denotes the source-predicted best short-epitope allele; these rows therefore evaluate post-vaccination, in-vitro restimulation responses and cannot establish untreated intrinsic immunogenicity or natural tumour presentation. + ### Training-overlap policy The official PRIME2 supplementary archive and Table S4 workbook were downloaded by stable article identifier and checksum [@gfeller2023prime2]. We compared canonical peptides, peptide–HLA pairs and labels against every benchmark record and indexed same-HLA, same-length Hamming-distance-one training neighbors. Because the public BigMHC construction incorporates non-random PRIME1/2 peptide records of eligible length into immunogenicity training/validation, those exact matches were flagged separately [@albert2023bigmhc]. A single union exclusion of all exact PRIME2 peptide–HLA matches defined the common benchmark because peptide specificity is HLA-conditioned; peptides seen only with another HLA received a separate exclusion sensitivity, as did one-substitution neighbors. PRIME2 Table S4 lacks source mutation, patient and study identifiers, so those overlap dimensions are explicitly unavailable rather than assumed absent. The full benchmark and row-level audit remain versioned. @@ -56,7 +58,7 @@ To distinguish fixed pretrained tools from models with known splitting, we fitte ### Resource packaging, sensitivity and quality control -The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. +The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. A machine-validated extension contract defines Dataset Cards, Predictor Cards and prediction artifacts, while the expanded artifact census preserves successful, non-comparable and failed public-tool attempts. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Cross-dataset stability and first-place probabilities are explicitly exploratory and task-stratified. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. ## Results @@ -119,12 +121,31 @@ The prospectively frozen primary metric was patient-macro NDCG@5 because the med | DeepImmuno-CNN | 1,015 | 0.526 | 0.158 | 0.755 (0.691–0.816) | 0.759 | -0.004 | | PRIME | 2,310 | 0.531 | 0.148 | 0.604 (0.546–0.660) | 0.581 | 0.023 | +### Endpoint-distinct RCC vaccine cohort + +The separately frozen RCC protocol retained 129 individually assayed short peptides from 9 vaccinated patients after excluding one source row with no usable short peptide/HLA assignment [@braun2025rcc]. The assay compared three peptide-stimulation replicates with three matched no-stimulation replicates; labels follow the source p-value threshold and therefore are assay-context outcomes rather than untreated biological negatives. No exact PRIME2, BigMHC-construction or DeepImmuno training overlap was identified among the 129 records, while DeepHLApan row-level training identity remains unknown. + +On near-complete support, PRIME had AUROC 0.580 and patient NDCG@5 0.691; BigMHC had 0.476 and 0.533, respectively. DeepImmuno-CNN supported only 51 records. With nine patients, all estimates are descriptive and do not establish a cross-domain interaction, universal ordering, natural tumour presentation or clinical efficacy. + +**Table 4. RCC personalized-vaccine cohort.** The endpoint is post-vaccination, individual-peptide IFN-γ ELISpot after in-vitro stimulation. Random NDCG@5 is calculated on each model's exact support. + +| Predictor | Predicted records | AUROC | AP | Patient NDCG@5 (95% CI) | Random NDCG@5 | Gain over random | +|---|---:|---:|---:|---:|---:|---:| +| BigMHC | 128 | 0.476 | 0.586 | 0.533 (0.418–0.640) | 0.587 | -0.054 | +| DeepHLApan | 128 | 0.505 | 0.653 | 0.614 (0.407–0.765) | 0.587 | 0.027 | +| DeepImmuno-CNN | 51 | 0.472 | 0.583 | 0.711 (0.591–0.839) | 0.695 | 0.016 | +| PRIME | 128 | 0.580 | 0.639 | 0.691 (0.611–0.789) | 0.587 | 0.104 | + +### Expanded reproducibility profile and extension contract + +The artifact census now records 12 pinned predictor entries. Beyond the five benchmarked tools, 7 entries are retained as profile-only, non-comparable, pending or unreproducible outcomes rather than being silently omitted. The public extension contract supplies machine-validated Dataset Cards, Predictor Cards and prediction-artifact schemas together with a common-support evaluator; these additions improve reuse but do not make heterogeneous prediction tasks scientifically interchangeable. + ### Reusable benchmark outputs NeoRepro's primary output is the versioned evidence chain rather than a winner label. Each benchmark record can be traced to a source row and overlap classification; each score records predictor version, task, direction and status; model comparisons use explicit common support; and all reported numerical outputs are generated from frozen result files. The resource also preserves the fully overlapped TESLA fixture and failed NCI eligibility gate as negative controls, so future evaluations can test whether leakage and invalid-label safeguards behave as intended. Metric validation agreed with an independent implementation to floating-point precision, and clean-checkout reproduction regenerated the frozen outputs. -Figures 1–5 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity and the vaccine-cohort extension, respectively (`results/figures/`). +Figures 1–6 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity, the Zhao vaccine-cohort extension and exploratory endpoint/domain stability, respectively (`results/figures/` and `results/analysis/stability/`). ## Discussion @@ -132,11 +153,11 @@ NeoRepro's central contribution is a reusable measurement contract. The first se The benchmark's biological results are deliberately secondary to that contract. On the filtered IMPROVE common set, PRIME showed better pooled and patient-level point estimates than BigMHC within the same broad peptide–HLA immunogenicity-score category. Their training labels and score contracts differ, so this is an observation about the pinned implementations and evaluation contract, not evidence of universal superiority. MHCflurry addresses presentation rather than T-cell recognition, was invoked without flanking context, and was tested only for association with the recognition endpoint; its results cannot validate presentation performance. All absolute average-precision values were low in a highly imbalanced screen, and patient-level pMHC retrieval varied widely. -The independent vaccine cohort illustrates why support and task context belong in the resource. BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of the cohort, and its high marginal NDCG@5 did not exceed the higher random expectation on that restricted support. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, but Zhao AUROC did not resolve their difference, IMPROVE NDCG@5 did not resolve the opposite direction, no formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. +The vaccine cohorts illustrate why support and task context belong in the resource. In Zhao, BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of Zhao and RCC, and high marginal NDCG@5 on restricted support cannot be read as a general advantage. RCC was smaller still, with only nine patients. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, whereas RCC and IMPROVE produced different point-estimate patterns. No formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. The transparent baselines refine interpretation of HLA effects. HLA-only models exceeded chance modestly under LOPO but weakened under LOSO, consistent with cohort- and allele-associated label structure. Peptide features supplied more stable signal; adding HLA did not consistently improve beyond them. For BigMHC, 32.5% of observed score variance lay between HLA groups, but this unadjusted quantity also mixes peptide composition, patient, cohort and preselection; it is not an isolated allele effect. HLA sensitivity therefore requires more than one shortcut diagnostic. -This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao peptides were likewise selected using earlier binding predictors. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. One positive-bearing patient in the source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five tools are a judicious executable subset, not an exhaustive census; they cover only MHC-I and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extension is biologically complementary rather than a pure replication: vaccination and peptide-pulsed dendritic-cell administration can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. +This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao and RCC peptides were likewise selected within vaccine-design workflows. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support the original IMPROVE LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. RCC adds only nine vaccinated patients and uses predicted HLA assignments. One positive-bearing patient in the IMPROVE source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five benchmarked tools are a judicious executable subset; the broader census includes heterogeneous tools that cannot be placed into the same comparison. All cover only MHC-I here and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extensions are biologically complementary rather than pure replications: vaccination and ex-vivo or in-vitro stimulation can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. The practical implication is that neoantigen benchmarking should release source-grounded record identifiers, predictor versions, complete missingness, training-overlap audits, support-matched references and patient-level retrieval alongside pooled metrics. NeoRepro provides these components as an executable resource that can be extended with new datasets or predictors without erasing negative results. Its value is not a new ranking algorithm or a universal leaderboard; it is a reproducible way to determine which comparison claims survive provenance, leakage, support and patient-level checks. diff --git a/paper/manuscript_template.md b/paper/manuscript_template.md index 96ffc5c..c416868 100644 --- a/paper/manuscript_template.md +++ b/paper/manuscript_template.md @@ -10,7 +10,7 @@ Public availability of a neoantigen predictor does not ensure that its software, ### Methods -We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and an independent personalized-vaccine ELISPOT cohort; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. +We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and two endpoint-distinct personalized-vaccine ELISPOT cohorts; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. ### Results @@ -32,7 +32,7 @@ NeoRepro asks a narrower question than developing a new predictor: what evidence ### Study design and reproducibility contract -The core protocol was frozen in `RESEARCH_SPEC.md`; the external extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json` before extension inference. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. +The core protocol was frozen in `RESEARCH_SPEC.md`; the Zhao extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json`, and the RCC cohort in `research/extension_protocol_rcc_v1.json`, before their respective inference runs. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. ### Dataset construction and provenance @@ -42,6 +42,8 @@ For the independent extension, we extracted checksum-pinned `Table1.xlsx` from t The administered peptides had been selected by the source workflow using earlier NetMHC/NetMHCpan binding predictions and an IC50 threshold below 500 nM [@zhao2026vaccine]. Consequently, this cohort evaluates reranking within a predictor-enriched candidate set rather than unselected tumor mutations. Reported peptide–HLA restrictions were computational assignments used by the source analysis and were not individually established by HLA-blocking experiments. +For the endpoint-distinct RCC extension, we checksum-pinned Supplementary Table 2 from Braun and colleagues and retained 129 short-peptide rows from nine vaccinated patients after excluding one row without a usable short peptide/HLA assignment [@braun2025rcc]. Every retained row contained three peptide-stimulation and three matched no-stimulation replicates. Positivity followed the source individual-peptide p-value rule. The HLA field denotes the source-predicted best short-epitope allele; these rows therefore evaluate post-vaccination, in-vitro restimulation responses and cannot establish untreated intrinsic immunogenicity or natural tumour presentation. + ### Training-overlap policy The official PRIME2 supplementary archive and Table S4 workbook were downloaded by stable article identifier and checksum [@gfeller2023prime2]. We compared canonical peptides, peptide–HLA pairs and labels against every benchmark record and indexed same-HLA, same-length Hamming-distance-one training neighbors. Because the public BigMHC construction incorporates non-random PRIME1/2 peptide records of eligible length into immunogenicity training/validation, those exact matches were flagged separately [@albert2023bigmhc]. A single union exclusion of all exact PRIME2 peptide–HLA matches defined the common benchmark because peptide specificity is HLA-conditioned; peptides seen only with another HLA received a separate exclusion sensitivity, as did one-substitution neighbors. PRIME2 Table S4 lacks source mutation, patient and study identifiers, so those overlap dimensions are explicitly unavailable rather than assumed absent. The full benchmark and row-level audit remain versioned. @@ -56,7 +58,7 @@ To distinguish fixed pretrained tools from models with known splitting, we fitte ### Resource packaging, sensitivity and quality control -The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. +The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. A machine-validated extension contract defines Dataset Cards, Predictor Cards and prediction artifacts, while the expanded artifact census preserves successful, non-comparable and failed public-tool attempts. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Cross-dataset stability and first-place probabilities are explicitly exploratory and task-stratified. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. ## Results @@ -66,7 +68,7 @@ The resource stores canonical schemas, source-row provenance, predictor and data NeoRepro's primary output is the versioned evidence chain rather than a winner label. Each benchmark record can be traced to a source row and overlap classification; each score records predictor version, task, direction and status; model comparisons use explicit common support; and all reported numerical outputs are generated from frozen result files. The resource also preserves the fully overlapped TESLA fixture and failed NCI eligibility gate as negative controls, so future evaluations can test whether leakage and invalid-label safeguards behave as intended. Metric validation agreed with an independent implementation to floating-point precision, and clean-checkout reproduction regenerated the frozen outputs. -Figures 1–5 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity and the vaccine-cohort extension, respectively (`results/figures/`). +Figures 1–6 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity, the Zhao vaccine-cohort extension and exploratory endpoint/domain stability, respectively (`results/figures/` and `results/analysis/stability/`). ## Discussion @@ -74,11 +76,11 @@ NeoRepro's central contribution is a reusable measurement contract. The first se The benchmark's biological results are deliberately secondary to that contract. On the filtered IMPROVE common set, PRIME showed better pooled and patient-level point estimates than BigMHC within the same broad peptide–HLA immunogenicity-score category. Their training labels and score contracts differ, so this is an observation about the pinned implementations and evaluation contract, not evidence of universal superiority. MHCflurry addresses presentation rather than T-cell recognition, was invoked without flanking context, and was tested only for association with the recognition endpoint; its results cannot validate presentation performance. All absolute average-precision values were low in a highly imbalanced screen, and patient-level pMHC retrieval varied widely. -The independent vaccine cohort illustrates why support and task context belong in the resource. BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of the cohort, and its high marginal NDCG@5 did not exceed the higher random expectation on that restricted support. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, but Zhao AUROC did not resolve their difference, IMPROVE NDCG@5 did not resolve the opposite direction, no formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. +The vaccine cohorts illustrate why support and task context belong in the resource. In Zhao, BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of Zhao and RCC, and high marginal NDCG@5 on restricted support cannot be read as a general advantage. RCC was smaller still, with only nine patients. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, whereas RCC and IMPROVE produced different point-estimate patterns. No formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. The transparent baselines refine interpretation of HLA effects. HLA-only models exceeded chance modestly under LOPO but weakened under LOSO, consistent with cohort- and allele-associated label structure. Peptide features supplied more stable signal; adding HLA did not consistently improve beyond them. For BigMHC, 32.5% of observed score variance lay between HLA groups, but this unadjusted quantity also mixes peptide composition, patient, cohort and preselection; it is not an isolated allele effect. HLA sensitivity therefore requires more than one shortcut diagnostic. -This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao peptides were likewise selected using earlier binding predictors. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. One positive-bearing patient in the source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five tools are a judicious executable subset, not an exhaustive census; they cover only MHC-I and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extension is biologically complementary rather than a pure replication: vaccination and peptide-pulsed dendritic-cell administration can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. +This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao and RCC peptides were likewise selected within vaccine-design workflows. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support the original IMPROVE LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. RCC adds only nine vaccinated patients and uses predicted HLA assignments. One positive-bearing patient in the IMPROVE source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five benchmarked tools are a judicious executable subset; the broader census includes heterogeneous tools that cannot be placed into the same comparison. All cover only MHC-I here and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extensions are biologically complementary rather than pure replications: vaccination and ex-vivo or in-vitro stimulation can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. The practical implication is that neoantigen benchmarking should release source-grounded record identifiers, predictor versions, complete missingness, training-overlap audits, support-matched references and patient-level retrieval alongside pooled metrics. NeoRepro provides these components as an executable resource that can be extended with new datasets or predictors without erasing negative results. Its value is not a new ranking algorithm or a universal leaderboard; it is a reproducible way to determine which comparison claims survive provenance, leakage, support and patient-level checks. diff --git a/paper/references.bib b/paper/references.bib index f078800..5b2edb1 100644 --- a/paper/references.bib +++ b/paper/references.bib @@ -134,3 +134,13 @@ @article{paul2020benchmark pmid = {32453790}, pmcid = {PMC7274474} } +@article{braun2025rcc, + author = {Braun, David A. and others}, + title = {A neoantigen vaccine generates antitumour immunity in renal cell carcinoma}, + journal = {Nature}, + year = {2025}, + volume = {639}, + pages = {474--482}, + doi = {10.1038/s41586-024-08507-5}, + url = {https://doi.org/10.1038/s41586-024-08507-5} +} diff --git a/predictors/example/adapter.py b/predictors/example/adapter.py new file mode 100644 index 0000000..176ee8f --- /dev/null +++ b/predictors/example/adapter.py @@ -0,0 +1,17 @@ +"""Minimal third-party adapter template. Replace only `predict` and keep the artifact contract.""" +import csv, sys + +def predict(peptide: str, hla: str) -> float: + # TODO: call the pinned upstream predictor; never return a fabricated score. + raise NotImplementedError + +def main(input_csv, output_csv): + with open(input_csv, newline="", encoding="utf-8") as src, open(output_csv, "w", newline="", encoding="utf-8") as dst: + rows=list(csv.DictReader(src)); fields=["record_id","predictor","predictor_version","task","score","score_direction","status"] + out=csv.DictWriter(dst, fieldnames=fields); out.writeheader() + for row in rows: + try: score=predict(row["peptide"], row["hla"]); status="predicted" + except Exception: score=""; status="failed" + out.writerow({"record_id":row["record_id"],"predictor":"YOUR_ID","predictor_version":"YOUR_VERSION","task":"YOUR_TASK","score":score,"score_direction":"higher","status":status}) + +if __name__ == "__main__": main(sys.argv[1], sys.argv[2]) diff --git a/predictors/mhcmatch/attempts/20260821T065057715285Z-test.json b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.json new file mode 100644 index 0000000..288c06e --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.json @@ -0,0 +1,16 @@ +{ + "command": [ + "predictors/mhcmatch/.venv/bin/mhcmatch", + "--help" + ], + "duration_seconds": 1.92031, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "mhcmatch", + "python": "3.11.15", + "returncode": 0, + "stage": "test", + "started_at": "2026-08-21T06:50:57.715285+00:00", + "stderr_log": "20260821T065057715285Z-test.stderr.log", + "stdout_log": "20260821T065057715285Z-test.stdout.log", + "timed_out": false +} diff --git a/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stderr.log b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stdout.log b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stdout.log new file mode 100644 index 0000000..9344186 --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stdout.log @@ -0,0 +1,49 @@ +usage: mhcmatch [-h] + {decompose,restriction,affinity,binder,scan,source,logo,span,predict,bootstrap,rank,explain,complement,neoag,mimicry,mimics,vector,deslip,expression} + ... + +peptide-MHC presentation tools + +positional arguments: + {decompose,restriction,affinity,binder,scan,source,logo,span,predict,bootstrap,rank,explain,complement,neoag,mimicry,mimics,vector,deslip,expression} + decompose split a peptide into anchor / TCR-facing parts (X + masks) + restriction rank presenting alleles for a peptide + affinity predict IC50 (nM) + neoantigen amplitude/DAI for a + peptide + binder generalized binder score (presentation x affinity) + ranked over alleles + scan find presented peptides in a protein (sequence or + FASTA path) + source find the self peptide a neoantigen derives from + logo motif logo (information content) + length distribution + span extend an MHC-II binding core to the full presented + ligand + predict score a variant peptide-window FASTA -> native + + .scored.csv + bootstrap pre-fetch the pmhc panel (and optionally proteomes) + from HF + rank rank neoantigen candidates (FASTA of windows, or a + scored table) + explain every component of the aggregate for one (peptide, + allele) + complement complementarity score (recognition axis) for peptides + — vectorised + neoag annotate candidates against the tested-neoantigen + database (nearest validated-immunogenic peptide + + substitution distance) + mimicry the fitted mimicry aggregate: signed viral / self / + thymus contributions per anchor and TCR-facing + channel, and their sum + mimics near-identical reference peptides per category (self / + thymus / viral / bacterial / neoag) -- batched and + threaded + vector assemble a polyepitope cassette: withdraw on safety, + choose how many units per allotype, order them, pick a + spacer + deslip find (and repair) the m1-pseudouridine +1 frameshift + motif in a cassette coding sequence + expression reference expression by normal tissue or tumour type + +options: + -h, --help show this help message and exit diff --git a/predictors/mhcmatch/attempts/20260821T065145972187Z-run.json b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.json new file mode 100644 index 0000000..f1e3ea8 --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.json @@ -0,0 +1,17 @@ +{ + "command": [ + "predictors/mhcmatch/.venv/bin/mhcmatch", + "binder", + "NLVPMVATV" + ], + "duration_seconds": 180.143593, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "mhcmatch", + "python": "3.11.15", + "returncode": 124, + "stage": "run", + "started_at": "2026-08-21T06:51:45.972187+00:00", + "stderr_log": "20260821T065145972187Z-run.stderr.log", + "stdout_log": "20260821T065145972187Z-run.stdout.log", + "timed_out": true +} diff --git a/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stderr.log b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stderr.log new file mode 100644 index 0000000..31ae88f --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stderr.log @@ -0,0 +1,3 @@ +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. + +Timed out after 180 seconds. \ No newline at end of file diff --git a/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stdout.log b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/clone.stderr.log b/predictors/mhcmatch/attempts/clone.stderr.log new file mode 100644 index 0000000..5743f41 --- /dev/null +++ b/predictors/mhcmatch/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcmatch/source'... diff --git a/predictors/mhcmatch/attempts/clone.stdout.log b/predictors/mhcmatch/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/install.stderr.log b/predictors/mhcmatch/attempts/install.stderr.log new file mode 100644 index 0000000..0fcd9cb --- /dev/null +++ b/predictors/mhcmatch/attempts/install.stderr.log @@ -0,0 +1,26 @@ +Using Python 3.11.15 environment at: predictors/mhcmatch/.venv +Resolved 18 packages in 290ms + Building mhcmatch @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcmatch/source +Downloading seqtree (1.5MiB) + Downloaded seqtree + Built mhcmatch @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcmatch/source +Prepared 3 packages in 366ms +Installed 18 packages in 70ms + + anyio==4.14.2 + + certifi==2026.7.22 + + click==8.4.2 + + filelock==3.32.3 + + fsspec==2026.7.0 + + h11==0.16.0 + + hf-xet==1.6.0 + + httpcore==1.0.9 + + httpx==0.28.1 + + huggingface-hub==1.28.0 + + idna==3.19 + + mhcmatch==0.25.0 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcmatch/source) + + numpy==2.4.6 + + packaging==26.3 + + pyyaml==6.0.3 + + seqtree==0.7.0 + + tqdm==4.70.0 + + typing-extensions==4.16.0 diff --git a/predictors/mhcmatch/attempts/install.stdout.log b/predictors/mhcmatch/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/smoke.stderr.log b/predictors/mhcmatch/attempts/smoke.stderr.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/mhcmatch/attempts/smoke.stderr.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/mhcmatch/attempts/smoke.stdout.log b/predictors/mhcmatch/attempts/smoke.stdout.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/mhcmatch/attempts/smoke.stdout.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/mhcmatch/attempts/sweep_receipt.json b/predictors/mhcmatch/attempts/sweep_receipt.json new file mode 100644 index 0000000..21e458f --- /dev/null +++ b/predictors/mhcmatch/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "mhcmatch", + "revision": "956bdb613fdb1e128cbd7d59fabb9b47ef1aaae9", + "repo": "https://github.com/antigenomics/mhcmatch.git", + "install_returncode": 0, + "smoke_returncode": 125, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/mhcnuggets/adapter.py b/predictors/mhcnuggets/adapter.py new file mode 100644 index 0000000..5b699fe --- /dev/null +++ b/predictors/mhcnuggets/adapter.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Strict peptide-HLA adapter for the pinned MHCnuggets BA models. + +MHCnuggets emits IC50 in nM; lower is better. Unsupported alleles are not +silently rescued to the closest allele. +""" +from __future__ import annotations +import argparse, csv, re, subprocess, tempfile +from pathlib import Path + +VERSION = "2.4.0@b666fea3" +FIELDS = ["record_id", "predictor", "predictor_version", "task", "score", "score_direction", "status", "raw_score", "provenance_path"] + +def allele(value: str) -> str: + compact = re.sub(r"[^A-Z0-9]", "", value.upper().replace("HLA-", "").replace("HLA", "")) + m = re.fullmatch(r"([ABC])([0-9]{2})([0-9]{2})", compact) + return f"HLA-{m.group(1)}{m.group(2)}:{m.group(3)}" if m else "" + +def main() -> int: + ap = argparse.ArgumentParser(); ap.add_argument("--input", type=Path, required=True); ap.add_argument("--source-dir", type=Path, default=Path("predictors/mhcnuggets/source")); ap.add_argument("--output", type=Path, required=True); args = ap.parse_args() + source = args.source_dir.resolve(); observed = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() + if observed != "b666fea3a54a1d357efba4ea4d8550ce5dd50aba": raise RuntimeError(f"unexpected revision {observed}") + rows = list(csv.DictReader(args.input.open(newline="", encoding="utf-8"))); required = {"record_id", "peptide", "hla"} + if not rows or required - set(rows[0]): raise ValueError(f"missing columns: {sorted(required-set(rows[0]))}") + out = {r["record_id"]: {"record_id": r["record_id"], "predictor": "MHCnuggets", "predictor_version": VERSION, "task": "binding", "score": "", "score_direction": "lower", "status": "unsupported_hla", "raw_score": "", "provenance_path": str(source)} for r in rows} + groups = {} + for r in rows: + h = allele(r["hla"]); model = source / "mhcnuggets" / "saves" / "production" / f"{h}_BA.h5" + if h and model.exists(): groups.setdefault(h, []).append(r) + with tempfile.TemporaryDirectory() as td: + for h, group in groups.items(): + pep = Path(td) / f"{h.replace(':','_')}.peps"; pred = Path(td) / f"{h.replace(':','_')}.csv"; pep.write_text("\n".join(r["peptide"] for r in group) + "\n", encoding="utf-8") + from mhcnuggets.src.predict import predict + predict(class_="I", peptides_path=str(pep), mhc=h, ba_models=True, output=str(pred)) + with pred.open(newline="", encoding="utf-8") as handle: + for r, p in zip(group, csv.DictReader(handle), strict=True): out[r["record_id"]].update(score=p["ic50"], raw_score=p["ic50"], status="predicted") + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", newline="", encoding="utf-8") as handle: w = csv.DictWriter(handle, fieldnames=FIELDS, lineterminator="\n"); w.writeheader(); w.writerows(out.values()) + print(f"MHCnuggets {VERSION}: {sum(r['status']=='predicted' for r in out.values())}/{len(out)} predicted -> {args.output}"); return 0 +if __name__ == "__main__": raise SystemExit(main()) diff --git a/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.json b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.json new file mode 100644 index 0000000..d3ba42c --- /dev/null +++ b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.json @@ -0,0 +1,22 @@ +{ + "command": [ + "predictors/mhcnuggets/.venv/bin/python", + "predictors/mhcnuggets/adapter.py", + "--input", + "predictors/mhcnuggets/fixture.csv", + "--source-dir", + "predictors/mhcnuggets/source", + "--output", + "predictors/mhcnuggets/fixture_predictions.csv" + ], + "duration_seconds": 7.534415, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "mhcnuggets", + "python": "3.11.15", + "returncode": 0, + "stage": "adapter", + "started_at": "2026-08-21T06:50:01.940493+00:00", + "stderr_log": "20260821T065001940493Z-adapter.stderr.log", + "stdout_log": "20260821T065001940493Z-adapter.stdout.log", + "timed_out": false +} diff --git a/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stderr.log b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stderr.log new file mode 100644 index 0000000..e0aed8b --- /dev/null +++ b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stderr.log @@ -0,0 +1,2 @@ +/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/.venv/lib/python3.11/site-packages/keras/src/layers/core/masking.py:48: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. + super().__init__(**kwargs) diff --git a/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stdout.log b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stdout.log new file mode 100644 index 0000000..b190a12 --- /dev/null +++ b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stdout.log @@ -0,0 +1,7 @@ +Predicting for 3 peptides +Number of peptides skipped/total due to length 0 / 0 +Building model +Closest allele found HLA-A02:01 +Predicting with only binding affinity trained models +Writing output files... +MHCnuggets 2.4.0@b666fea3: 3/3 predicted -> predictors/mhcnuggets/fixture_predictions.csv diff --git a/predictors/mhcnuggets/attempts/clone.stderr.log b/predictors/mhcnuggets/attempts/clone.stderr.log new file mode 100644 index 0000000..3f1c808 --- /dev/null +++ b/predictors/mhcnuggets/attempts/clone.stderr.log @@ -0,0 +1,105 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source'... +Updating files: 0% (2/354) +Updating files: 1% (4/354) +Updating files: 2% (8/354) +Updating files: 3% (11/354) +Updating files: 4% (15/354) +Updating files: 5% (18/354) +Updating files: 5% (20/354) +Updating files: 6% (22/354) +Updating files: 7% (25/354) +Updating files: 8% (29/354) +Updating files: 9% (32/354) +Updating files: 10% (36/354) +Updating files: 11% (39/354) +Updating files: 12% (43/354) +Updating files: 13% (47/354) +Updating files: 14% (50/354) +Updating files: 15% (54/354) +Updating files: 16% (57/354) +Updating files: 17% (61/354) +Updating files: 18% (64/354) +Updating files: 19% (68/354) +Updating files: 20% (71/354) +Updating files: 21% (75/354) +Updating files: 22% (78/354) +Updating files: 23% (82/354) +Updating files: 24% (85/354) +Updating files: 25% (89/354) +Updating files: 26% (93/354) +Updating files: 27% (96/354) +Updating files: 28% (100/354) +Updating files: 29% (103/354) +Updating files: 30% (107/354) +Updating files: 31% (110/354) +Updating files: 32% (114/354) +Updating files: 33% (117/354) +Updating files: 34% (121/354) +Updating files: 35% (124/354) +Updating files: 36% (128/354) +Updating files: 37% (131/354) +Updating files: 38% (135/354) +Updating files: 39% (139/354) +Updating files: 40% (142/354) +Updating files: 41% (146/354) +Updating files: 42% (149/354) +Updating files: 43% (153/354) +Updating files: 44% (156/354) +Updating files: 45% (160/354) +Updating files: 46% (163/354) +Updating files: 47% (167/354) +Updating files: 48% (170/354) +Updating files: 48% (171/354) +Updating files: 49% (174/354) +Updating files: 50% (177/354) +Updating files: 51% (181/354) +Updating files: 52% (185/354) +Updating files: 53% (188/354) +Updating files: 54% (192/354) +Updating files: 55% (195/354) +Updating files: 56% (199/354) +Updating files: 57% (202/354) +Updating files: 58% (206/354) +Updating files: 59% (209/354) +Updating files: 60% (213/354) +Updating files: 61% (216/354) +Updating files: 62% (220/354) +Updating files: 63% (224/354) +Updating files: 64% (227/354) +Updating files: 65% (231/354) +Updating files: 66% (234/354) +Updating files: 67% (238/354) +Updating files: 68% (241/354) +Updating files: 69% (245/354) +Updating files: 70% (248/354) +Updating files: 71% (252/354) +Updating files: 72% (255/354) +Updating files: 73% (259/354) +Updating files: 74% (262/354) +Updating files: 75% (266/354) +Updating files: 76% (270/354) +Updating files: 77% (273/354) +Updating files: 78% (277/354) +Updating files: 79% (280/354) +Updating files: 80% (284/354) +Updating files: 81% (287/354) +Updating files: 82% (291/354) +Updating files: 83% (294/354) +Updating files: 84% (298/354) +Updating files: 85% (301/354) +Updating files: 86% (305/354) +Updating files: 87% (308/354) +Updating files: 88% (312/354) +Updating files: 89% (316/354) +Updating files: 90% (319/354) +Updating files: 91% (323/354) +Updating files: 92% (326/354) +Updating files: 93% (330/354) +Updating files: 94% (333/354) +Updating files: 95% (337/354) +Updating files: 96% (340/354) +Updating files: 97% (344/354) +Updating files: 98% (347/354) +Updating files: 99% (351/354) +Updating files: 100% (354/354) +Updating files: 100% (354/354), done. diff --git a/predictors/mhcnuggets/attempts/clone.stdout.log b/predictors/mhcnuggets/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcnuggets/attempts/install.stderr.log b/predictors/mhcnuggets/attempts/install.stderr.log new file mode 100644 index 0000000..6aa1123 --- /dev/null +++ b/predictors/mhcnuggets/attempts/install.stderr.log @@ -0,0 +1,8 @@ +Using Python 3.11.15 environment at: predictors/mhcnuggets/.venv +Resolved 56 packages in 590ms + Building mhcnuggets @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcnuggets/source + Built mhcnuggets @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcnuggets/source +Prepared 1 package in 246ms +Uninstalled 1 package in 2ms +Installed 1 package in 2ms + ~ mhcnuggets==2.4.0 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcnuggets/source) diff --git a/predictors/mhcnuggets/attempts/install.stdout.log b/predictors/mhcnuggets/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcnuggets/attempts/smoke.stderr.log b/predictors/mhcnuggets/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcnuggets/attempts/smoke.stdout.log b/predictors/mhcnuggets/attempts/smoke.stdout.log new file mode 100644 index 0000000..40a891b --- /dev/null +++ b/predictors/mhcnuggets/attempts/smoke.stdout.log @@ -0,0 +1 @@ +import ok diff --git a/predictors/mhcnuggets/attempts/sweep_receipt.json b/predictors/mhcnuggets/attempts/sweep_receipt.json new file mode 100644 index 0000000..a90f7d6 --- /dev/null +++ b/predictors/mhcnuggets/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "mhcnuggets", + "revision": "b666fea3a54a1d357efba4ea4d8550ce5dd50aba", + "repo": "https://github.com/KarchinLab/mhcnuggets.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/mhcnuggets/fixture.csv b/predictors/mhcnuggets/fixture.csv new file mode 100644 index 0000000..f5b7d6b --- /dev/null +++ b/predictors/mhcnuggets/fixture.csv @@ -0,0 +1,4 @@ +record_id,peptide,hla +mhcnuggets-fixture-1,AIAACAMLLV,HLA-A*02:01 +mhcnuggets-fixture-2,ALVCYIVMPV,HLA-A*02:01 +mhcnuggets-fixture-3,ALEPRKEIDV,HLA-A*02:01 diff --git a/predictors/mhcnuggets/fixture_predictions.csv b/predictors/mhcnuggets/fixture_predictions.csv new file mode 100644 index 0000000..9e173ec --- /dev/null +++ b/predictors/mhcnuggets/fixture_predictions.csv @@ -0,0 +1,4 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,raw_score,provenance_path +mhcnuggets-fixture-1,MHCnuggets,2.4.0@b666fea3,binding,49.76,lower,predicted,49.76,/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source +mhcnuggets-fixture-2,MHCnuggets,2.4.0@b666fea3,binding,13.9,lower,predicted,13.9,/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source +mhcnuggets-fixture-3,MHCnuggets,2.4.0@b666fea3,binding,5707.79,lower,predicted,5707.79,/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source diff --git a/predictors/neofox/attempts/20260821T065057715299Z-test.json b/predictors/neofox/attempts/20260821T065057715299Z-test.json new file mode 100644 index 0000000..0cd2d46 --- /dev/null +++ b/predictors/neofox/attempts/20260821T065057715299Z-test.json @@ -0,0 +1,16 @@ +{ + "command": [ + "predictors/neofox/.venv/bin/neofox", + "--help" + ], + "duration_seconds": 41.265039, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "neofox", + "python": "3.11.15", + "returncode": 0, + "stage": "test", + "started_at": "2026-08-21T06:50:57.715299+00:00", + "stderr_log": "20260821T065057715299Z-test.stderr.log", + "stdout_log": "20260821T065057715299Z-test.stdout.log", + "timed_out": false +} diff --git a/predictors/neofox/attempts/20260821T065057715299Z-test.stderr.log b/predictors/neofox/attempts/20260821T065057715299Z-test.stderr.log new file mode 100644 index 0000000..fa570c8 --- /dev/null +++ b/predictors/neofox/attempts/20260821T065057715299Z-test.stderr.log @@ -0,0 +1,4 @@ +/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neofox/source/neofox/model/conversion.py:203: SyntaxWarning: assertion is always true, perhaps remove parentheses? + assert(mhc in [MHC_I, MHC_II], 'Bad MHC value') +/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neofox/source/neofox/model/conversion.py:260: SyntaxWarning: assertion is always true, perhaps remove parentheses? + assert (mhc in [MHC_I, MHC_II], 'Bad MHC value') diff --git a/predictors/neofox/attempts/20260821T065057715299Z-test.stdout.log b/predictors/neofox/attempts/20260821T065057715299Z-test.stdout.log new file mode 100644 index 0000000..49ea27c --- /dev/null +++ b/predictors/neofox/attempts/20260821T065057715299Z-test.stdout.log @@ -0,0 +1,47 @@ +usage: neofox [-h] --input-file INPUT_FILE --patient-data PATIENTS_DATA + --output-folder OUTPUT_FOLDER [--output-prefix OUTPUT_PREFIX] + [--with-all-neoepitopes] + [--rank-mhci-threshold RANK_MHCI_THRESHOLD] + [--rank-mhcii-threshold RANK_MHCII_THRESHOLD] + [--num-cpus NUM_CPUS] [--config CONFIG] + [--organism {human,mouse}] [--verbose] + +NeoFox 1.2.3 annotates a given set of neoantigen candidate sequences derived +from point mutation with relevant neoantigen features + +options: + -h, --help show this help message and exit + --input-file INPUT_FILE + Input file with neoantigens candidates represented by + long mutated peptide sequences. Supported formats: + tab-separated columns (extensions: .txt or .tsv) or + JSON (extension: .json) + --patient-data PATIENTS_DATA + file with data for patients with columns: identifier, + estimated_tumor_content, mhc_i_alleles, + mhc_ii_alleles, tissue + --output-folder OUTPUT_FOLDER + output folder + --output-prefix OUTPUT_PREFIX + prefix to name output files in the output folder + --with-all-neoepitopes + output annotations for all MHC-I and MHC-II + neoepitopes on all HLA alleles + --rank-mhci-threshold RANK_MHCI_THRESHOLD + MHC-I epitopes with a netMHCpan predicted rank greater + than or equal than this threshold will be filtered out + (default: 2.0) + --rank-mhcii-threshold RANK_MHCII_THRESHOLD + MHC-II epitopes with a netMHCIIpan predicted rank + greater than or equal than this threshold will be + filtered out (default: 5.0) + --num-cpus NUM_CPUS number of CPUs for computation + --config CONFIG an optional configuration file with all the + environment variables + --organism {human,mouse} + the organism to which the data corresponds + --verbose verbose logs + +NeoFox (NEOantigen Feature toolbOX) 1.2.3. Copyright (c) 2020-2021 TRON - +Translational Oncology at the University Medical Center of the Johannes +Gutenberg University Mainz gGmbH, all rights reserved diff --git a/predictors/neofox/attempts/clone.stderr.log b/predictors/neofox/attempts/clone.stderr.log new file mode 100644 index 0000000..5e7eb7d --- /dev/null +++ b/predictors/neofox/attempts/clone.stderr.log @@ -0,0 +1,103 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neofox/source'... +Updating files: 0% (2/329) +Updating files: 1% (4/329) +Updating files: 2% (7/329) +Updating files: 3% (10/329) +Updating files: 4% (14/329) +Updating files: 5% (17/329) +Updating files: 6% (20/329) +Updating files: 7% (24/329) +Updating files: 8% (27/329) +Updating files: 9% (30/329) +Updating files: 10% (33/329) +Updating files: 11% (37/329) +Updating files: 12% (40/329) +Updating files: 13% (43/329) +Updating files: 14% (47/329) +Updating files: 15% (50/329) +Updating files: 16% (53/329) +Updating files: 17% (56/329) +Updating files: 18% (60/329) +Updating files: 19% (63/329) +Updating files: 20% (66/329) +Updating files: 21% (70/329) +Updating files: 22% (73/329) +Updating files: 23% (76/329) +Updating files: 24% (79/329) +Updating files: 25% (83/329) +Updating files: 26% (86/329) +Updating files: 27% (89/329) +Updating files: 28% (93/329) +Updating files: 29% (96/329) +Updating files: 30% (99/329) +Updating files: 31% (102/329) +Updating files: 32% (106/329) +Updating files: 33% (109/329) +Updating files: 34% (112/329) +Updating files: 35% (116/329) +Updating files: 36% (119/329) +Updating files: 37% (122/329) +Updating files: 38% (126/329) +Updating files: 39% (129/329) +Updating files: 40% (132/329) +Updating files: 41% (135/329) +Updating files: 42% (139/329) +Updating files: 43% (142/329) +Updating files: 44% (145/329) +Updating files: 45% (149/329) +Updating files: 46% (152/329) +Updating files: 47% (155/329) +Updating files: 48% (158/329) +Updating files: 49% (162/329) +Updating files: 50% (165/329) +Updating files: 51% (168/329) +Updating files: 52% (172/329) +Updating files: 53% (175/329) +Updating files: 54% (178/329) +Updating files: 55% (181/329) +Updating files: 56% (185/329) +Updating files: 57% (188/329) +Updating files: 58% (191/329) +Updating files: 59% (195/329) +Updating files: 60% (198/329) +Updating files: 61% (201/329) +Updating files: 62% (204/329) +Updating files: 63% (208/329) +Updating files: 64% (211/329) +Updating files: 65% (214/329) +Updating files: 66% (218/329) +Updating files: 67% (221/329) +Updating files: 68% (224/329) +Updating files: 69% (228/329) +Updating files: 70% (231/329) +Updating files: 71% (234/329) +Updating files: 72% (237/329) +Updating files: 73% (241/329) +Updating files: 74% (244/329) +Updating files: 75% (247/329) +Updating files: 76% (251/329) +Updating files: 77% (254/329) +Updating files: 78% (257/329) +Updating files: 79% (260/329) +Updating files: 80% (264/329) +Updating files: 81% (267/329) +Updating files: 82% (270/329) +Updating files: 83% (274/329) +Updating files: 84% (277/329) +Updating files: 85% (280/329) +Updating files: 86% (283/329) +Updating files: 87% (287/329) +Updating files: 88% (290/329) +Updating files: 89% (293/329) +Updating files: 90% (297/329) +Updating files: 91% (300/329) +Updating files: 92% (303/329) +Updating files: 93% (306/329) +Updating files: 94% (310/329) +Updating files: 95% (313/329) +Updating files: 96% (316/329) +Updating files: 97% (320/329) +Updating files: 98% (323/329) +Updating files: 99% (326/329) +Updating files: 100% (329/329) +Updating files: 100% (329/329), done. diff --git a/predictors/neofox/attempts/clone.stdout.log b/predictors/neofox/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neofox/attempts/install.stderr.log b/predictors/neofox/attempts/install.stderr.log new file mode 100644 index 0000000..ea9c9ce --- /dev/null +++ b/predictors/neofox/attempts/install.stderr.log @@ -0,0 +1,8 @@ +Using Python 3.11.15 environment at: predictors/neofox/.venv +Resolved 44 packages in 356ms + Building neofox @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/neofox/source + Built neofox @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/neofox/source +Prepared 1 package in 269ms +Uninstalled 1 package in 7ms +Installed 1 package in 18ms + ~ neofox==1.2.3 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/neofox/source) diff --git a/predictors/neofox/attempts/install.stdout.log b/predictors/neofox/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neofox/attempts/smoke.stderr.log b/predictors/neofox/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neofox/attempts/smoke.stdout.log b/predictors/neofox/attempts/smoke.stdout.log new file mode 100644 index 0000000..40a891b --- /dev/null +++ b/predictors/neofox/attempts/smoke.stdout.log @@ -0,0 +1 @@ +import ok diff --git a/predictors/neofox/attempts/sweep_receipt.json b/predictors/neofox/attempts/sweep_receipt.json new file mode 100644 index 0000000..9152fd9 --- /dev/null +++ b/predictors/neofox/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "neofox", + "revision": "dd894a67ad02a4377ed3e4b1e44bd576360c98cd", + "repo": "https://github.com/TRON-Bioinformatics/neofox.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/neoguider/attempts/clone.stderr.log b/predictors/neoguider/attempts/clone.stderr.log new file mode 100644 index 0000000..61986ef --- /dev/null +++ b/predictors/neoguider/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neoguider/source'... diff --git a/predictors/neoguider/attempts/clone.stdout.log b/predictors/neoguider/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neoguider/attempts/install.stderr.log b/predictors/neoguider/attempts/install.stderr.log new file mode 100644 index 0000000..4c76f1c --- /dev/null +++ b/predictors/neoguider/attempts/install.stderr.log @@ -0,0 +1,5 @@ +Using Python 3.11.15 environment at: predictors/neoguider/.venv +error: /Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research + Project/predictors/neoguider/source does not appear to be a Python + project, as neither `pyproject.toml` nor `setup.py` are present in the + directory diff --git a/predictors/neoguider/attempts/install.stdout.log b/predictors/neoguider/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neoguider/attempts/smoke.stderr.log b/predictors/neoguider/attempts/smoke.stderr.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/neoguider/attempts/smoke.stderr.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/neoguider/attempts/smoke.stdout.log b/predictors/neoguider/attempts/smoke.stdout.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/neoguider/attempts/smoke.stdout.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/neoguider/attempts/sweep_receipt.json b/predictors/neoguider/attempts/sweep_receipt.json new file mode 100644 index 0000000..4790412 --- /dev/null +++ b/predictors/neoguider/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "neoguider", + "revision": "f95a5713453910902d657f1d148568b93c81026f", + "repo": "https://github.com/XuegongLab/neoguider.git", + "install_returncode": 2, + "smoke_returncode": 125, + "license_files": [ + "LICENSE.txt" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/pvactools/attempts/clone.stderr.log b/predictors/pvactools/attempts/clone.stderr.log new file mode 100644 index 0000000..22e82b4 --- /dev/null +++ b/predictors/pvactools/attempts/clone.stderr.log @@ -0,0 +1,110 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/pvactools/source'... +Updating files: 0% (2/4089) +Updating files: 1% (41/4089) +Updating files: 2% (82/4089) +Updating files: 3% (123/4089) +Updating files: 4% (164/4089) +Updating files: 5% (205/4089) +Updating files: 6% (246/4089) +Updating files: 6% (247/4089) +Updating files: 7% (287/4089) +Updating files: 8% (328/4089) +Updating files: 9% (369/4089) +Updating files: 10% (409/4089) +Updating files: 11% (450/4089) +Updating files: 12% (491/4089) +Updating files: 13% (532/4089) +Updating files: 14% (573/4089) +Updating files: 15% (614/4089) +Updating files: 16% (655/4089) +Updating files: 17% (696/4089) +Updating files: 18% (737/4089) +Updating files: 19% (777/4089) +Updating files: 20% (818/4089) +Updating files: 21% (859/4089) +Updating files: 22% (900/4089) +Updating files: 23% (941/4089) +Updating files: 24% (982/4089) +Updating files: 25% (1023/4089) +Updating files: 26% (1064/4089) +Updating files: 26% (1093/4089) +Updating files: 27% (1105/4089) +Updating files: 28% (1145/4089) +Updating files: 29% (1186/4089) +Updating files: 30% (1227/4089) +Updating files: 31% (1268/4089) +Updating files: 31% (1274/4089) +Updating files: 32% (1309/4089) +Updating files: 33% (1350/4089) +Updating files: 34% (1391/4089) +Updating files: 35% (1432/4089) +Updating files: 36% (1473/4089) +Updating files: 37% (1513/4089) +Updating files: 38% (1554/4089) +Updating files: 39% (1595/4089) +Updating files: 40% (1636/4089) +Updating files: 41% (1677/4089) +Updating files: 42% (1718/4089) +Updating files: 43% (1759/4089) +Updating files: 44% (1800/4089) +Updating files: 45% (1841/4089) +Updating files: 46% (1881/4089) +Updating files: 47% (1922/4089) +Updating files: 48% (1963/4089) +Updating files: 49% (2004/4089) +Updating files: 50% (2045/4089) +Updating files: 51% (2086/4089) +Updating files: 52% (2127/4089) +Updating files: 53% (2168/4089) +Updating files: 54% (2209/4089) +Updating files: 55% (2249/4089) +Updating files: 56% (2290/4089) +Updating files: 57% (2331/4089) +Updating files: 57% (2352/4089) +Updating files: 58% (2372/4089) +Updating files: 59% (2413/4089) +Updating files: 60% (2454/4089) +Updating files: 61% (2495/4089) +Updating files: 62% (2536/4089) +Updating files: 63% (2577/4089) +Updating files: 64% (2617/4089) +Updating files: 65% (2658/4089) +Updating files: 66% (2699/4089) +Updating files: 67% (2740/4089) +Updating files: 68% (2781/4089) +Updating files: 69% (2822/4089) +Updating files: 70% (2863/4089) +Updating files: 71% (2904/4089) +Updating files: 72% (2945/4089) +Updating files: 73% (2985/4089) +Updating files: 74% (3026/4089) +Updating files: 75% (3067/4089) +Updating files: 76% (3108/4089) +Updating files: 77% (3149/4089) +Updating files: 78% (3190/4089) +Updating files: 79% (3231/4089) +Updating files: 80% (3272/4089) +Updating files: 81% (3313/4089) +Updating files: 82% (3353/4089) +Updating files: 83% (3394/4089) +Updating files: 84% (3435/4089) +Updating files: 85% (3476/4089) +Updating files: 86% (3517/4089) +Updating files: 87% (3558/4089) +Updating files: 88% (3599/4089) +Updating files: 89% (3640/4089) +Updating files: 90% (3681/4089) +Updating files: 91% (3721/4089) +Updating files: 91% (3748/4089) +Updating files: 92% (3762/4089) +Updating files: 93% (3803/4089) +Updating files: 93% (3808/4089) +Updating files: 94% (3844/4089) +Updating files: 95% (3885/4089) +Updating files: 96% (3926/4089) +Updating files: 97% (3967/4089) +Updating files: 97% (3975/4089) +Updating files: 98% (4008/4089) +Updating files: 99% (4049/4089) +Updating files: 100% (4089/4089) +Updating files: 100% (4089/4089), done. diff --git a/predictors/pvactools/attempts/clone.stdout.log b/predictors/pvactools/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/pvactools/attempts/install.stderr.log b/predictors/pvactools/attempts/install.stderr.log new file mode 100644 index 0000000..2174ec3 --- /dev/null +++ b/predictors/pvactools/attempts/install.stderr.log @@ -0,0 +1,8 @@ +Using Python 3.11.15 environment at: predictors/pvactools/.venv +Resolved 107 packages in 748ms + Building pvactools @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/pvactools/source + Built pvactools @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/pvactools/source +Prepared 1 package in 607ms +Uninstalled 1 package in 2ms +Installed 1 package in 5ms + ~ pvactools==7.1.2 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/pvactools/source) diff --git a/predictors/pvactools/attempts/install.stdout.log b/predictors/pvactools/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/pvactools/attempts/smoke.stderr.log b/predictors/pvactools/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/pvactools/attempts/smoke.stdout.log b/predictors/pvactools/attempts/smoke.stdout.log new file mode 100644 index 0000000..38a22ff --- /dev/null +++ b/predictors/pvactools/attempts/smoke.stdout.log @@ -0,0 +1,49 @@ +usage: pvacseq [-h] + {run,binding_filter,coverage_filter,transcript_filter,top_score_filter,aggregate_report_filter,net_chop,netmhc_stab,calculate_reference_proteome_similarity,generate_protein_fasta,create_peptide_ordering_form,generate_aggregated_report,identify_problematic_amino_acids,mark_genes_of_interest,add_ml_predictions,update_tiers,download_example_data,install_vep_plugin} + ... + +positional arguments: + {run,binding_filter,coverage_filter,transcript_filter,top_score_filter,aggregate_report_filter,net_chop,netmhc_stab,calculate_reference_proteome_similarity,generate_protein_fasta,create_peptide_ordering_form,generate_aggregated_report,identify_problematic_amino_acids,mark_genes_of_interest,add_ml_predictions,update_tiers,download_example_data,install_vep_plugin} + run Run the pVACseq pipeline. + binding_filter Filter variants processed by IEDB by binding score. + coverage_filter Filter variants processed by IEDB by coverage, vaf, + and gene expression. + transcript_filter Filter variant transcripts processed by IEDB. + top_score_filter Pick the best neoepitope for each variant. + aggregate_report_filter + Filter an aggregate report and its metrics.json file + based on the variant Tier. + net_chop Run NetChop on existing pVACseq output .tsv to predict + cleavage sites on the neoepitopes. + netmhc_stab Run NetMHCStabPan on existing pVACseq output .tsv to + add stability predictions to the neoepitopes. + calculate_reference_proteome_similarity + Blast peptides against the reference proteome on + existing pVACseq output .tsv. + generate_protein_fasta + Generate an annotated fasta file from a VCF with + protein sequences of mutations and matching wildtypes. + create_peptide_ordering_form + Generate a peptide ordering form with coloring. + generate_aggregated_report + Generate an aggregated report from a pVACseq + .all_epitopes.tsv report file. + identify_problematic_amino_acids + Mark problematic amino acid positions in each epitope + or filter entries that have problematic amino acids. + mark_genes_of_interest + Mark predictions resulting from variants on a genes of + interest list. + add_ml_predictions Add ML-based neoantigen evaluation predictions to + pVACtools output files. + update_tiers Update tiers in an aggregated report in order to, for + example, use different thresholds or account for + problematic position or reference match information if + run after initial pipeline run. + download_example_data + Download example input and output files. + install_vep_plugin Install the Wildtype and Frameshift VEP plugins into + your VEP_plugins directory. + +options: + -h, --help show this help message and exit diff --git a/predictors/pvactools/attempts/sweep_receipt.json b/predictors/pvactools/attempts/sweep_receipt.json new file mode 100644 index 0000000..6cadce2 --- /dev/null +++ b/predictors/pvactools/attempts/sweep_receipt.json @@ -0,0 +1,13 @@ +{ + "predictor": "pvactools", + "revision": "2ce871be5ae56af141fe723d60d3841125874a74", + "repo": "https://github.com/griffithlab/pVACtools.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE", + "docs/license.rst" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/seq2neo/attempts/clone.stderr.log b/predictors/seq2neo/attempts/clone.stderr.log new file mode 100644 index 0000000..9c0bd54 --- /dev/null +++ b/predictors/seq2neo/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/seq2neo/source'... diff --git a/predictors/seq2neo/attempts/clone.stdout.log b/predictors/seq2neo/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/seq2neo/attempts/install.stderr.log b/predictors/seq2neo/attempts/install.stderr.log new file mode 100644 index 0000000..90236d5 --- /dev/null +++ b/predictors/seq2neo/attempts/install.stderr.log @@ -0,0 +1,9 @@ +Using Python 3.11.15 environment at: predictors/seq2neo/.venv + × No solution found when resolving dependencies: + ╰─▶ Because tensorflow==2.3.0 has no wheels with a matching Python ABI tag + (e.g., `cp311`) and seq2neo==2.1 depends on tensorflow==2.3.0, we can + conclude that seq2neo==2.1 cannot be used. + And because only seq2neo==2.1 is available and you require seq2neo, we + can conclude that your requirements are unsatisfiable. + +hint: You require CPython 3.11 (`cp311`), but we only found wheels for `tensorflow` (v2.3.0) with the following Python ABI tags: `cp35m`, `cp36m`, `cp37m`, `cp38` diff --git a/predictors/seq2neo/attempts/install.stdout.log b/predictors/seq2neo/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/seq2neo/attempts/smoke.stderr.log b/predictors/seq2neo/attempts/smoke.stderr.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/seq2neo/attempts/smoke.stderr.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/seq2neo/attempts/smoke.stdout.log b/predictors/seq2neo/attempts/smoke.stdout.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/seq2neo/attempts/smoke.stdout.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/seq2neo/attempts/sweep_receipt.json b/predictors/seq2neo/attempts/sweep_receipt.json new file mode 100644 index 0000000..7259312 --- /dev/null +++ b/predictors/seq2neo/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "seq2neo", + "revision": "f342aa4ffa9012fb5509914333b9401a1eb286c4", + "repo": "https://github.com/XSLiuLab/Seq2Neo.git", + "install_returncode": 1, + "smoke_returncode": 125, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/vaxrank/attempts/clone.stderr.log b/predictors/vaxrank/attempts/clone.stderr.log new file mode 100644 index 0000000..2835688 --- /dev/null +++ b/predictors/vaxrank/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/vaxrank/source'... diff --git a/predictors/vaxrank/attempts/clone.stdout.log b/predictors/vaxrank/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/vaxrank/attempts/install.stderr.log b/predictors/vaxrank/attempts/install.stderr.log new file mode 100644 index 0000000..3d64e42 --- /dev/null +++ b/predictors/vaxrank/attempts/install.stderr.log @@ -0,0 +1,106 @@ +Using Python 3.11.15 environment at: predictors/vaxrank/.venv +Resolved 90 packages in 1.65s + Building vaxrank @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/vaxrank/source + Building matplotlib-venn==1.1.2 + Building docopt==0.6.2 +Downloading astropy (6.3MiB) +Downloading pyphen (2.0MiB) +Downloading oncoref (2.8MiB) + Downloaded pyphen + Built docopt==0.6.2 + Downloaded oncoref + Built vaxrank @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/vaxrank/source + Built matplotlib-venn==1.1.2 + Downloaded astropy +Prepared 25 packages in 552ms +Installed 90 packages in 917ms + + adjusttext==1.4.0 + + appdirs==1.4.4 + + argcomplete==3.7.2 + + astropy==8.0.1 + + astropy-iers-data==0.2026.8.18.14.22.31 + + biopython==1.88 + + brotli==1.2.0 + + certifi==2026.7.22 + + cffi==2.1.1 + + charset-normalizer==3.5.1 + + contourpy==1.3.3 + + cssselect2==0.9.0 + + cycler==0.12.1 + + datacache==1.7.0 + + dnachisel==3.2.16 + + docopt==0.6.2 + + et-xmlfile==2.0.0 + + filelock==3.32.3 + + flametree==0.2.1 + + fonttools==4.63.0 + + fsspec==2026.7.0 + + gtfparse==2.8.0 + + idna==3.19 + + isovar==1.7.2 + + jinja2==3.1.6 + + joblib==1.5.3 + + kiwisolver==1.5.0 + + markupsafe==3.0.3 + + matplotlib==3.11.1 + + matplotlib-venn==1.1.2 + + memoized-property==1.0.3 + + mhcflurry==2.2.1 + + mhcgnomes==3.33.5 + + mhcnames==0.4.8 + + mhctools==3.31.5 + + mpmath==1.3.0 + + msgspec==0.21.1 + + narwhals==2.25.0 + + networkx==3.6.1 + + numpy==2.4.6 + + oncoref==1.8.174 + + openpyxl==3.1.5 + + packaging==26.3 + + pandas==2.3.3 + + pdfkit==1.0.0 + + pillow==12.3.0 + + platformdirs==4.11.3 + + polars==1.43.2 + + polars-runtime-32==1.43.2 + + proglog==0.1.12 + + psutil==7.2.2 + + pyarrow==25.0.1 + + pycparser==3.0 + + pydyf==0.12.1 + + pyensembl==2.10.4 + + pyerfa==2.0.1.5 + + pyparsing==3.3.2 + + pyphen==0.18.1 + + pysam==0.24.0 + + python-codon-tables==0.1.18 + + python-dateutil==2.9.0.post0 + + pytz==2026.3.post1 + + pyyaml==6.0.3 + + requests==2.34.2 + + roman==5.2 + + scikit-learn==1.9.0 + + scipy==1.17.1 + + sercol==1.0.0 + + serializable==1.1.0 + + setuptools==84.0.0 + + simplejson==4.1.1 + + six==1.17.0 + + sympy==1.14.0 + + threadpoolctl==3.6.0 + + tinycss2==1.5.1 + + tinyhtml5==2.1.0 + + tinytimer==0.0.0 + + topiary==5.16.2 + + torch==2.13.0 + + tqdm==4.70.0 + + typechecks==0.1.0 + + typing-extensions==4.16.0 + + tzdata==2026.3 + + urllib3==2.7.0 + + varcode==5.0.6 + + vaxrank==3.1.14 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/vaxrank/source) + + weasyprint==69.0 + + webencodings==0.6.1 + + xvfbwrapper==0.2.29 + + zopfli==0.4.3 diff --git a/predictors/vaxrank/attempts/install.stdout.log b/predictors/vaxrank/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/vaxrank/attempts/smoke.stderr.log b/predictors/vaxrank/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/vaxrank/attempts/smoke.stdout.log b/predictors/vaxrank/attempts/smoke.stdout.log new file mode 100644 index 0000000..40a891b --- /dev/null +++ b/predictors/vaxrank/attempts/smoke.stdout.log @@ -0,0 +1 @@ +import ok diff --git a/predictors/vaxrank/attempts/sweep_receipt.json b/predictors/vaxrank/attempts/sweep_receipt.json new file mode 100644 index 0000000..3364844 --- /dev/null +++ b/predictors/vaxrank/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "vaxrank", + "revision": "5c71edccda920fafaab5986837d3dd87c8ce8911", + "repo": "https://github.com/openvax/vaxrank.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/reports/manuscript_resource_audit.json b/reports/manuscript_resource_audit.json index 36a5f18..d47e1f6 100644 --- a/reports/manuscript_resource_audit.json +++ b/reports/manuscript_resource_audit.json @@ -2,6 +2,7 @@ "citations_used": [ "albert2023bigmhc", "borch2024improve", + "braun2025rcc", "gfeller2023prime2", "kim2025tscape", "muller2023harmonized", diff --git a/reports/rcc_predictor_run.json b/reports/rcc_predictor_run.json new file mode 100644 index 0000000..ad82468 --- /dev/null +++ b/reports/rcc_predictor_run.json @@ -0,0 +1,108 @@ +{ + "artifacts": [ + { + "path": "results/raw_predictions/rcc/bigmhc-v1.0.csv", + "predictor": "BigMHC", + "rows": 129, + "sha256": "1b16e7bf545032a9763a9c899190775dddf24ea19378d8763899663a460f8dd5", + "version": "v1.0" + }, + { + "path": "results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "predictor": "DeepHLApan", + "rows": 129, + "sha256": "4d084a12522852fbdfe3bcbe9b64a8477f496b2073e6f8fc95943bfd18b3e4d4", + "version": "1.1.1@ac1f4beb" + }, + { + "path": "results/raw_predictions/rcc/deepimmuno-cnn.csv", + "predictor": "DeepImmuno-CNN", + "rows": 129, + "sha256": "be8fdb1f4efff6ef3159971a46f58513e8cc90d2d2f5737494aa14871700ad1e", + "version": "1.0@df42ac5b" + }, + { + "path": "results/raw_predictions/rcc/prime-2.0.csv", + "predictor": "PRIME", + "rows": 129, + "sha256": "68ed9f4f7a928964ccbab018d315e0e3ff9315d1b5c5a80665c30dee9a861140", + "version": "2.0" + } + ], + "benchmark": "data/processed/rcc_vaccine_benchmark.csv", + "completed_at": "2026-08-21T06:54:36.965390+00:00", + "orchestrator_python": "3.14.7 (main, Aug 7 2026, 02:15:30) [Clang 22.1.3 ]", + "parallel": true, + "platform": "macOS-15.3-arm64-arm-64bit-Mach-O", + "reuse_existing": false, + "runs": [ + { + "command": [ + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/.venv/bin/python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/source", + "--python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/.venv/bin/python", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/bigmhc-v1.0.csv" + ], + "execution": "executed", + "predictor": "bigmhc", + "returncode": 0, + "runtime_seconds": 65.49302654100029 + }, + { + "command": [ + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deephlapan/.venv/bin/python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deephlapan/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deephlapan/source", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deephlapan-1.1.1.csv" + ], + "execution": "executed", + "predictor": "deephlapan", + "returncode": 0, + "runtime_seconds": 51.63081233399862 + }, + { + "command": [ + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deepimmuno/.venv/bin/python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deepimmuno/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deepimmuno/source", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deepimmuno-cnn.csv" + ], + "execution": "executed", + "predictor": "deepimmuno", + "returncode": 0, + "runtime_seconds": 12.130533249997825 + }, + { + "command": [ + "/Users/steve/.local/share/uv/python/cpython-3.14-macos-aarch64-none/bin/python3.14", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/prime/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/prime/source", + "--mix-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/prime/vendor/mixmhcpred", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/prime-2.0.csv" + ], + "execution": "executed", + "predictor": "prime", + "returncode": 0, + "runtime_seconds": 2.97150474999944 + } + ] +} diff --git a/research/extension_protocol_rcc_v1.json b/research/extension_protocol_rcc_v1.json new file mode 100644 index 0000000..ffb5040 --- /dev/null +++ b/research/extension_protocol_rcc_v1.json @@ -0,0 +1,46 @@ +{ + "protocol_id": "neorepro-rcc-vaccine-extension-v1", + "frozen_at": "2026-08-20", + "status": "completed_prediction_and_evaluation", + "dataset": { + "name": "Braun et al. personalized RCC peptide vaccine cohort", + "article_doi": "10.1038/s41586-024-08507-5", + "supplement_member": "41586_2024_8507_MOESM4_ESM.xlsx", + "supplement_url": "https://media.springernature.com/original/springer-static/esm/art%3A10.1038%2Fs41586-024-08507-5/MediaObjects/41586_2024_8507_MOESM4_ESM.xlsx", + "supplement_sha256": "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1", + "worksheet": "sheet2", + "source_rows": 130, + "excluded_rows": 1, + "rows": 129, + "patients": 9, + "endpoint": "post-vaccination in-vitro peptide-stimulation IFN-gamma ELISpot", + "short_peptide_field": "Short_Epitope", + "hla_field": "HLA_of_best_short_epitope", + "patient_field": "Patient_ID", + "positive_rule": "source-reported individual-peptide p-value threshold; preserve continuous replicate values and source p-value", + "negative_rule": "not an untreated biological negative; row-level comparator is the matched no-stimulation control within the same assay", + "scope_boundary": "independent vaccine-elicited response domain; do not call untreated, natural presentation, clinical efficacy or intrinsic immunogenicity", + "eligibility_gate": [ + "129 retained rows have patient ID, short epitope and HLA field; one source row has N/A short epitope/HLA and is excluded", + "all 129 retained rows have three peptide-stimulation and three no-stimulation replicates", + "predicted HLA semantics are retained explicitly", + "no untested enumerated candidates are converted into negatives" + ] + }, + "prediction_policy": { + "status": "completed", + "eligible_tools": ["BigMHC v1.0", "PRIME 2.0", "DeepImmuno-CNN", "DeepHLApan"], + "exclude": ["MHCflurry from same-task primary because it is a presentation score"], + "primary": "patient-macro NDCG@5 on pairwise common prediction support", + "secondary": ["pooled AUROC", "average precision", "patient Recall@5", "coverage"], + "leakage": "apply known exact peptide-HLA union; unknown training identity remains unknown", + "reason_for_separate_protocol": "endpoint and treatment context differ from IMPROVE and Zhao", + "results": { + "BigMHC": {"predicted_rows": 128, "auroc": 0.476, "patient_ndcg5": 0.533}, + "PRIME": {"predicted_rows": 128, "auroc": 0.580, "patient_ndcg5": 0.691}, + "DeepHLApan": {"predicted_rows": 128, "auroc": 0.505, "patient_ndcg5": 0.614}, + "DeepImmuno-CNN": {"predicted_rows": 51, "auroc": 0.472, "patient_ndcg5": 0.711} + }, + "interpretation": "RCC is a small endpoint-distinct vaccine cohort; DeepImmuno-CNN has 39.5% coverage and all conclusions are descriptive, not universal model rankings." + } +} diff --git a/research/external_cohort_failure_protocols.json b/research/external_cohort_failure_protocols.json new file mode 100644 index 0000000..2eff722 --- /dev/null +++ b/research/external_cohort_failure_protocols.json @@ -0,0 +1,72 @@ +{ + "protocol_id": "neorepro-external-cohort-funnel-v1", + "created_at": "2026-08-20", + "status": "screening_complete_rcc_frozen_evx_failed", + "hard_gate": "Do not convert enumerated, untested, unreported, or merely computational candidates into experimental negatives.", + "screening_rule": [ + "Every retained row must be an individually assayed short peptide observation or a source-grounded explicit negative.", + "Patient identity, peptide, HLA and source study/cohort must be recoverable without guessing.", + "Treatment or vaccination context is recorded and cannot be relabeled untreated/natural presentation.", + "Training-set status is audited separately; unknown is not absence of overlap.", + "A source with incomplete evidence is pending, not eligible." + ], + "failures": [ + { + "candidate_id": "nci_mmp_2021", + "failure_code": "UNTESTED_ENUMERATED_CANDIDATES", + "evidence_url": "https://ndownloader.figshare.com/files/26040944", + "reason": "The archive enumerates short candidates derived from screened parent mutations, but does not establish individual testing for each short peptide-HLA row. Unknown rows cannot be labeled pMHC negatives.", + "action": "Preserve research/extension_protocol_v1_nci_failed.json; no prediction run." + }, + { + "candidate_id": "gbm_neovax_2025", + "failure_code": "NO_VERIFIED_COMPLETE_PEPTIDE_HLA_TABLE", + "evidence_url": "https://aacr.figshare.com/articles/dataset/Supplementary_Table_S5_from_Integrating_Multisector_Molecular_Characterization_into_Personalized_Peptide_Vaccine_Design_for_Patients_with_Newly_Diagnosed_Glioblastoma/29071987", + "reason": "Located public artifact is TCR-sequence data and does not itself verify complete patient-peptide-HLA experimental negatives.", + "action": "Do not download or infer missing rows; retain as failed until a source-grounded row-level table is found." + }, + { + "candidate_id": "melanoma_personal_2021", + "failure_code": "RESTRICTED_AND_ENDPOINT_INCOMPLETE", + "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC8273876/", + "reason": "Public supplements emphasize selected vaccine responses and the article states some patient-related data are subject to confidentiality/MTA; complete class-I patient-peptide-HLA experimental-negative rows are not established.", + "action": "Exclude from benchmark; do not treat unreported peptides as negatives." + }, + { + "candidate_id": "nepdb_2021", + "failure_code": "PROVENANCE_AND_TRAINING_UNCERTAIN", + "evidence_url": "https://doi.org/10.3389/fimmu.2021.644637", + "reason": "A curated database is not automatically a reconstructable patient-grouped cohort; patient IDs, assay denominators, cohort boundaries and training overlap are not verified at row level.", + "action": "Use only for discovery/provenance cross-checking, not as an external cohort." + }, + { + "candidate_id": "tesla_2020", + "failure_code": "KNOWN_TRAINING_OVERLAP", + "evidence_url": "https://doi.org/10.1016/j.cell.2020.09.015", + "reason": "The local audit found all 520 pilot rows exact peptide-HLA matches in official PRIME2 training and concordant with the published BigMHC immunogenicity construction.", + "action": "Retain only as leakage-positive control; no external-validity claim." + }, + { + "candidate_id": "itsndb_2023", + "failure_code": "PATIENT_AND_NEGATIVE_SEMANTICS_NOT_VERIFIED", + "evidence_url": "https://doi.org/10.3389/fimmu.2023.1094236", + "reason": "The public description mixes curated neoantigens with simulated prioritization scenarios; a complete patient-linked, individually tested negative table is not established.", + "action": "Do not promote to candidate without source-table audit." + }, + { + "candidate_id": "evx01_2024", + "failure_code": "NO_COMPLETE_SHORT_PEPTIDE_HLA_NEGATIVE_TABLE", + "evidence_url": "https://jitc.bmj.com/content/12/5/e008817", + "reason": "The study reports individual restimulation of long vaccine peptides, but the public evidence does not provide a complete short-peptide-HLA row table with experimentally defined pMHC negatives; irrelevant-peptide background is not equivalent to a source-grounded pMHC negative cohort.", + "action": "Exclude from benchmark and preserve the source-level failure." + } + ], + "pending": [], + "eligible_after_member_audit": [ + { + "candidate_id": "rcc_pcv_2025", + "protocol": "research/extension_protocol_rcc_v1.json", + "member_sha256": "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1" + } + ] +} diff --git a/research/external_cohort_funnel.csv b/research/external_cohort_funnel.csv new file mode 100644 index 0000000..4fd02b3 --- /dev/null +++ b/research/external_cohort_funnel.csv @@ -0,0 +1,11 @@ +candidate_id,title,year,source_url,source_type,individual_short_peptide_assay,patient_id,hla,experimental_negative,study_cohort_recoverable,redistribution,training_set_risk,eligibility,decision,evidence_note,accessed_at +improve_2024,IMPROVE feature model broad-scale validation of T-cell recognition,2024,https://doi.org/10.3389/fimmu.2024.1360281,paper+repository,yes,yes,yes,yes,yes,public repository terms require audit,unknown,eligible,existing_primary,"17,520 patient-matched screened records; already frozen as principal benchmark; exact PRIME2 overlap audited separately.",2026-08-20 +zhao_2026,Profiling immunogenic neoantigen peptides elicited by personalized neoantigen vaccine,2026,https://pmc.ncbi.nlm.nih.gov/articles/PMC13286890/,paper+Europe PMC supplement,yes,yes,yes,yes,yes,CC BY 4.0,partial,eligible,existing_external,"2,317 administered 8-11mer peptides from 352 patients; individual IFN-gamma ELISPOT; 313 source positives under ratio >=2.0; already frozen and run.",2026-08-20 +nci_mmp_2021,NCI Surgery Branch mutated minimal peptide test set,2021,https://ndownloader.figshare.com/files/26040944,Figshare archive,unknown,partial,yes,no,yes,CC BY 4.0,unknown,failed,failed_eligibility,"2.36-GB file enumerates short candidates derived from parent screens; row-level short-peptide experimental testing is not established, so unknown rows cannot be negatives.",2026-08-20 +rcc_pcv_2025,A neoantigen vaccine generates antitumour immunity in renal cell carcinoma,2025,https://doi.org/10.1038/s41586-024-08507-5,Nature supplement,yes,yes,yes,partial,yes,partial,CC BY-NC-ND 4.0,eligible,eligible_external,"Checksum-pinned Supplementary Table 2 sheet 2 has 130 source rows from 9 patients; 129 retained rows have canonical short epitope, predicted HLA and three peptide-stimulation plus three no-stimulation replicates. One N/A short-epitope/HLA row is excluded. Positives are defined by source p-value threshold; endpoint is post-vaccine ELISpot and therefore endpoint-distinct from untreated natural presentation.",2026-08-20 +evx01_2024,Dose escalation study of a personalized peptide-based neoantigen vaccine,2024,https://jitc.bmj.com/content/12/5/e008817,Nature/PMC supplement,yes,yes,partial,partial,yes,partial,CC BY-NC,failed,failed_eligibility,"The paper reports 91 individually restimulated long vaccine peptides and patient IDs, but the public evidence does not expose a complete row-level short-peptide-HLA table with experimental negatives; HLA/minimal epitopes are prediction-derived and responses use irrelevant-peptide background rather than a verified pMHC negative table.",2026-08-20 +gbm_neovax_2025,Integrating Multisector Molecular Characterization into Personalized Peptide Vaccine Design for GBM,2025,https://aacr.figshare.com/articles/dataset/Supplementary_Table_S5_from_Integrating_Multisector_Molecular_Characterization_into_Personalized_Peptide_Vaccine_Design_for_Patients_with_Newly_Diagnosed_Glioblastoma/29071987,Figshare supplement,partial,yes,unknown,partial,yes,CC BY 4.0,unknown,failed,failed_eligibility,"Figshare item is TCR-sequence data; article says IFN-gamma ELISPOT for patients 1-3, but the located artifact does not establish a complete row-level pMHC table with HLA and experimental negatives.",2026-08-20 +melanoma_personal_2021,Personal neoantigen vaccines induce persistent memory T-cell responses,2021,https://pmc.ncbi.nlm.nih.gov/articles/PMC8273876/,paper+supplement,partial,yes,partial,no,yes,restricted/MTA for some data,unknown,failed,failed_eligibility,"Supplementary data emphasize vaccine-positive responses, pools and class-II work; paper states patient-related data may require MTA and does not establish a complete individually tested class-I negative table.",2026-08-20 +nepdb_2021,NEPdb experimentally validated neoantigens and predicted neoepitopes,2021,https://doi.org/10.3389/fimmu.2021.644637,database,partial,unknown,yes,partial,partial,public site terms unclear,high,failed,failed_eligibility,"Curated records include effective/ineffective entries and HLA, but patient identity, assay denominator and source-cohort reconstruction are not guaranteed at record level; database/model-training overlap risk is high.",2026-08-20 +tesla_2020,TESLA consortium validation set,2020,https://doi.org/10.1016/j.cell.2020.09.015,paper+supplement,yes,yes,yes,yes,yes,source-specific terms,known,failed,failed_leakage_gate,"Experimentally tested patient-matched records are real, but all 520 retained pilot rows exactly overlap PRIME2 training and BigMHC immunogenicity construction; retained only as leakage-positive control.",2026-08-20 +itsndb_2023,Unraveling tumor-specific neoantigen immunogenicity prediction,2023,https://doi.org/10.3389/fimmu.2023.1094236,paper+database,partial,partial,yes,partial,partial,unknown,high,failed,failed_eligibility,"Includes curated presented neoantigens and simulated prioritization scenarios, but the full patient-linked experimental-negative contract needed here is not established from the public article/database description.",2026-08-20 diff --git a/research/research_log.md b/research/research_log.md index bd313e2..8ab60f8 100644 --- a/research/research_log.md +++ b/research/research_log.md @@ -87,3 +87,38 @@ - The Zhao source yielded 2,317 individually administered peptides from 352 patients (313 positives). Known exact training-overlap union exclusion removed two positive records; 2,315 records, 311 positives and 131 positive-bearing patients remained. - The frozen 2,000-replicate primary comparison found BigMHC above PRIME by 0.057 patient-macro NDCG@5 (95% CI 0.008–0.106) and above DeepHLApan by 0.078 (0.022–0.133) on near-complete common support. DeepImmuno-CNN covered 43.8%; its common-support differences were unresolved. - The BigMHC–PRIME direction reversed relative to IMPROVE. This is interpreted as model-by-domain dependence, not a universal-winner result. The vaccine-elicited post-vaccination ELISPOT endpoint remains distinct from natural tumor presentation and clinical benefit. + +## 2026-08-20 — External-cohort funnel audit (third/fourth queue) + +- Added `research/external_cohort_funnel.csv` and `research/external_cohort_failure_protocols.json` to preserve a source-by-source eligibility funnel rather than treating literature mentions as datasets. +- Screened ten candidates spanning IMPROVE, Zhao, NCI, RCC PCV, EVX-01, GBM NeoVax, melanoma personal vaccines, NEPdb, TESLA and ITSNdb. Zhao and IMPROVE remain the only currently verified eligible patient-grouped benchmark datasets in this repository; Zhao has already been frozen and run. +- NCI, GBM NeoVax, melanoma personal-vaccine data, NEPdb and ITSNdb received explicit failure reasons. NCI remains the decisive example of the mandatory rule: enumerated or untested short candidates are not experimental negatives. +- RCC PCV and EVX-01 remain `pending` rather than eligible because the public evidence establishes individual-peptide assay components but does not yet verify the complete patient–peptide–HLA row-level negative contract, exact HLA semantics and machine-readable redistribution path. +- No new predictions were launched from a pending source. The next safe action is a targeted supplement-member audit followed by a new frozen protocol only if every eligibility field passes. + +## 2026-08-20 — RCC member audit and EVX-01 exclusion + +- RCC Supplementary Table 2 was downloaded from the official Nature member URL and checksum-pinned (`c113c42b...5d0c1`). Worksheet 2 contains 130 rows from 9 patients; every row has a short epitope, predicted HLA field, three peptide-stimulation replicates and three no-stimulation replicates. RCC is therefore eligible as an endpoint-distinct vaccine cohort, and `research/extension_protocol_rcc_v1.json` was frozen before prediction. +- RCC is not untreated and its HLA values are predicted binding alleles; all downstream reporting must preserve those limitations. It cannot be merged with IMPROVE as natural presentation or with Zhao as an identical assay endpoint. +- EVX-01 was excluded after source-level audit. The paper reports 91 individually restimulated long vaccine peptides, but the public evidence does not provide a complete short-peptide-HLA row table with experimentally defined pMHC negatives; its background is an irrelevant peptide control. No prediction was run for EVX-01. + +## 2026-08-20 — RCC extension prediction and evaluation completed + +- Built `data/processed/rcc_vaccine_benchmark.csv` from 129 retained rows after excluding one N/A short-epitope/HLA source row; 9 patients, 75 positives and 54 negatives. Source checksum and row-level provenance are retained in `data/rcc_vaccine_summary.json` and the canonical rows. +- Training-overlap audit found zero exact PRIME2, published BigMHC construction, or DeepImmuno peptide-HLA overlaps; near-sequence results are retained, and DeepHLApan training identity remains unknown. +- Fixed predictors ran with coverage: BigMHC 128/129, PRIME 128/129, DeepHLApan 128/129, DeepImmuno-CNN 51/129. Missing outputs were not imputed or treated as negatives. +- Existing evaluation produced descriptive pooled AUROC / patient NDCG@5: BigMHC 0.476 / 0.533, PRIME 0.580 / 0.691, DeepHLApan 0.505 / 0.614, DeepImmuno-CNN 0.472 / 0.711. Unequal support, vaccine endpoint, predicted-HLA semantics and small patient count prohibit universal ranking claims. + +## 2026-08-20 — Public predictor reproduction sweep + +- Added `scripts/reproduce_public_predictors.py` to clone each queued public repository, capture the immutable HEAD revision and license-file paths, create a per-predictor Python 3.11 environment, run editable installation, and preserve install/smoke stdout, stderr, and JSON receipts. +- MHCnuggets, NeoFox, pVACtools, Vaxrank, and mhcmatch installed successfully. MHCnuggets and NeoFox passed import smoke tests; pVACtools and Vaxrank passed `--help`. mhcmatch has no documented smoke entry point in the checked revision. +- NeoGuider did not expose a Python package entry point (`pyproject.toml`/`setup.py` absent), and Seq2Neo requires TensorFlow 2.3.0, which has no Python 3.11 wheel. These are recorded as reproducibility failures, not silently repaired. +- pVACtools, NeoFox, Vaxrank, and Seq2Neo remain profile-only or non-comparable because their output contracts are end-to-end/annotation/ranking workflows rather than the canonical peptide–HLA score contract. No new candidate entered the benchmark. +- Follow-up MHCnuggets audit: production BA weights and curated training tables are present in the pinned repository. The official three-peptide example ran successfully under Python 3.11/TensorFlow and the strict adapter produced 3/3 binding scores with lower-IC50-is-better semantics. The repository's `saves/test/HLA-A01:01_test_model` is a separate test checkpoint whose expected values cannot validate the production checkpoint; this is recorded as a limitation rather than an exact-match claim. The predictor remains pending for binding-track inclusion until training-overlap and benchmark-task alignment are audited. +- Efficiency correction: mhcmatch's first binder invocation triggered its expensive calibration/reference bootstrap and timed out at 180 seconds. The cheap local CLI/API smoke path (`--help`, `decompose`) passed; the timeout is retained as evidence of an external/reference-data dependency, and no further large bootstrap was attempted for this profile-only candidate. +## 2026-08-20 — Cross-dataset stability and model-selection-risk analysis + +Implemented `scripts/analyze_stability.py` and generated `results/analysis/stability/` from the frozen IMPROVE and Zhao benchmark/prediction artifacts. The analysis is explicitly exploratory/descriptive heterogeneity analysis: it produces a dataset × predictor × metric matrix, record-level Spearman ranking concordance, patient-bootstrap probability of being first at Recall@5, BigMHC–PRIME direction-reversal probability, and coverage-threshold common-support summaries. It uses analytic fixed-score rankings and a recorded seed; it does not perform post-hoc significance testing, causal inference, or clinical efficacy claims. The script preserves predictor status metadata and writes `analysis_metadata.json` with limitations and configuration. + +Acceptance follow-up: added explicit `leave_one_domain_out.csv`, `endpoint_domain_metadata.csv`, a dependency-free endpoint/domain AUROC SVG, and `tests/test_stability_analysis.py`. First-place probabilities are task-stratified so presentation and immunogenicity predictors are not placed in one leaderboard. The expensive 2,000-bootstrap output remains the authoritative model-selection file; the leave-one-domain-out and visualization artifacts are deterministic post-processing of the frozen inputs. diff --git a/research/training_overlap_audit_rcc.csv b/research/training_overlap_audit_rcc.csv new file mode 100644 index 0000000..5ae946c --- /dev/null +++ b/research/training_overlap_audit_rcc.csv @@ -0,0 +1,130 @@ +record_id,benchmark_label,exact_prime2_peptide_hla,exact_bigmhc_im_trainval,exact_deepimmuno_peptide_hla,union_known_exact_overlap,prime2_label_conflict,deepimmuno_label_conflict,peptide_only_prime2_different_hla,peptide_only_deepimmuno_different_hla,near_hamming1_prime2_same_hla,near_hamming1_deepimmuno_same_hla,deephlapan_training_overlap +rcc-vaccine-8c2e1ec6d8f675cf,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6a6ffb8b81ec9208,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4adeedbe0e35cf09,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e06799bdf8de8af5,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-969998ab9d8b4dc1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a5952e9e71bd27bd,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6fa3e9ad450e030e,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-8a9041cc55b08e38,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e11c81feedc5751e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-fefffdcccca8649c,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-940b62df9c3078c4,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-43742d14e082c2e6,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-769c6b305827c570,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e74f183589095f61,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-15ab70cb934aad8f,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c8a8ce5ab15f9d4b,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f05602883865f4bf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b92105a12fb6f028,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a004075aed453bcf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-70eb710f90e2973b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c81528933be4a147,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-2a2f0b8f959aa3f6,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-74e8a99b7ef86c61,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-88bea2527dd72a1d,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ff6bbb8d69468a47,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-8ea6c4f21d2c81ee,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b2f6110d808ac0da,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f43e93564087f160,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-fc1276f53c3f45e1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-3e6b1c1d8de0ce71,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4255bdded9e984bd,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-31c3a5a1c962db43,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-d3d4046d155047b6,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-5e4096b57e93c92e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e8d6aeb51705642a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-26dc24e186221fe2,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-364822c022710958,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9c477d13a3359f5e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-487a8e43d12c17ac,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-539b50b223ea0ee1,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-618d6aa7d8232657,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-71f5d2ab19a389e8,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-409a0661a319bbb1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9f431d040990f0f7,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7c1009a0bb0524b3,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-42e869aba90dc4bd,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a1a88d1c11b4fcdd,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-d42b9fb3a44366ac,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ce806d8b1a4132b2,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ea51c18cc4e16995,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9bce007ba3a4aff0,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-aa3019acbdddb7dc,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-1bcdaed331509a08,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4a1853322eea7a14,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-cde49959d75e6ec2,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7f8e1f8e02332663,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b8aa4d97d214f765,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-5993797f7a6c7057,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-eacb2f310488ca9d,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-70f05fcfa038224f,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4d15c4e62bb19311,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a6b2180415ff1b56,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e52f3609d694b4ba,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-763f29ff7ac4c65d,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c331f65899396b9b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-09c9b699d977f421,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e944d5a782a7d074,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-75f86cf28dca60e2,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-5b2ef450eaf2248a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c50e7f0e1b3b69a5,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ae70f68909b113e8,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-efdf21635c142ab8,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-817b1ac6328997a7,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-feb66d3c66c8a4de,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7946e9a10fe114cf,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f3e32e857dd09d71,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-240bf5b232815c6a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-40c01a17a3ed4a2e,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7b823cc728ea1450,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-0c7eb5e3295f0ad2,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-88aaf2311ce63196,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-bcae618cfad77c1f,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-11f61d47947266cc,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f4a4db2362e7a54e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-1eceb5b6a969d721,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-55106745d24cfbf4,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a808947689fbe42c,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-590f5ea08f24e2cf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b984dc3be5c1fa17,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6ff6e758ac279425,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-aa0d4e0503dacba4,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4d44829f31cf5558,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4f7d9f9f22d48440,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-60d1ba1b8ae350fd,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7ec85836cb09ee30,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4b7fdcc54f65b98b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-d02de725ecc685a5,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e1d984ebf245728f,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a9a3c2bc98a74746,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-da610226e338a8de,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ca0a8a8bb4e2d094,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c45585f253341070,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b2761351ccb757b7,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b1ad936ccc2f249a,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-02261377bb800592,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-834aff6d2a7dc363,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-0036b8c45a4cd20d,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-08447808dd0300dd,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-34c3e7e7a2dcd8b6,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-1ae968c5ae01f49b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-29f7c92d094c1ced,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-0aaef5904890371c,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-27b47d43dab2b353,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-46b486b9d199e93a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b8a9c44bcf53568f,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7425139a6230adc0,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e39c49c350f42009,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-73717d4619b66faf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f130cbb5ea1ca0fa,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-3f16099c911d2791,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-07bc8054e1edd65a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-99bd603a759d6b22,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ad726d8ef91bd0e6,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-972e1f28fe4cb96a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e0d628668c1eccf2,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-3d165d772065bb3a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9ef49682993d2ea1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6c01a27a6a575007,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-12b152d35907e280,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable diff --git a/research/training_overlap_summary_rcc.json b/research/training_overlap_summary_rcc.json new file mode 100644 index 0000000..89e2c02 --- /dev/null +++ b/research/training_overlap_summary_rcc.json @@ -0,0 +1,23 @@ +{ + "benchmark": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark_full.csv", + "benchmark_exact_bigmhc_im_trainval": 0, + "benchmark_exact_deepimmuno": 0, + "benchmark_exact_prime2": 0, + "benchmark_near_deepimmuno": 0, + "benchmark_near_prime2": 0, + "benchmark_rows": 129, + "benchmark_union_known_exact_overlap": 0, + "deepimmuno_training_rows_resolved": 9055, + "deepimmuno_training_rows_source": 9055, + "deepimmuno_unresolved_hla_rows": 0, + "overlap_dimensions": { + "bigmhc_published_immunogenicity_construction": "checked_via_prime2_random_flag", + "deephlapan_training_identity": "unknown_official_row_manifest_unavailable", + "deepimmuno_exact_peptide_hla": "checked", + "near_sequence_same_hla_same_length": "checked_for_prime2_and_deepimmuno", + "patient_and_study_training_identity": "unavailable", + "prime2_exact_peptide_hla": "checked" + }, + "prime2_training_rows": 65585, + "prime2_workbook_sha256": "641a104764167f9f04bafb6606e519e5625740ed1720af7d15ac9026636bc23a" +} diff --git a/results/analysis/rcc/metrics.json b/results/analysis/rcc/metrics.json new file mode 100644 index 0000000..9ca2681 --- /dev/null +++ b/results/analysis/rcc/metrics.json @@ -0,0 +1,2911 @@ +{ + "benchmark": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "common_support": [ + { + "left": "BigMHC", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "task": "immunogenicity" + }, + { + "left": "BigMHC", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "task": "immunogenicity" + }, + { + "left": "BigMHC", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "task": "immunogenicity" + }, + { + "left": "DeepHLApan", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "task": "immunogenicity" + }, + { + "left": "DeepHLApan", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "task": "immunogenicity" + }, + { + "left": "DeepImmuno-CNN", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "task": "immunogenicity" + } + ], + "config": { + "bootstrap": 2000, + "ks": [ + 5, + 10, + 20 + ], + "seed": 20260820 + }, + "metrics": { + "BigMHC": { + "hla": { + "HLA-A*01:01": { + "auroc": 0.5555555555555556, + "average_precision": 0.9467813051146384, + "brier": 0.7766608713597656, + "n": 10, + "positives": 9 + }, + "HLA-A*02:01": { + "auroc": 0.8055555555555556, + "average_precision": 0.5888888888888888, + "brier": 0.17650530414755602, + "n": 9, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.6666666666666666, + "average_precision": 0.75, + "brier": 0.2554600718003685, + "n": 5, + "positives": 2 + }, + "HLA-A*25:01": { + "auroc": null, + "average_precision": null, + "brier": 5.337579490387601e-05, + "n": 1, + "positives": 0 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.009819210166266542, + "n": 3, + "positives": 0 + }, + "HLA-A*31:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.4452767978083257, + "n": 2, + "positives": 1 + }, + "HLA-A*32:01": { + "auroc": 0.0, + "average_precision": 0.5833333333333333, + "brier": 0.6360571360656988, + "n": 3, + "positives": 2 + }, + "HLA-A*66:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.4588287991777727, + "n": 4, + "positives": 2 + }, + "HLA-B*07:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.27139674555417276, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.5333333333333333, + "average_precision": 0.7250000000000001, + "brier": 0.32692333515866856, + "n": 8, + "positives": 5 + }, + "HLA-B*13:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.13057078448859577, + "n": 2, + "positives": 1 + }, + "HLA-B*18:01": { + "auroc": 0.25, + "average_precision": 0.5833333333333333, + "brier": 0.5087623762373769, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.5, + "average_precision": 0.5381944444444444, + "brier": 0.2843966461375668, + "n": 10, + "positives": 4 + }, + "HLA-B*27:06": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9825847383470074, + "n": 1, + "positives": 1 + }, + "HLA-B*35:01": { + "auroc": null, + "average_precision": null, + "brier": 0.039145639455604904, + "n": 1, + "positives": 0 + }, + "HLA-B*35:03": { + "auroc": 0.0, + "average_precision": 0.3333333333333333, + "brier": 0.3219427027809941, + "n": 3, + "positives": 1 + }, + "HLA-B*38:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.39194105610017, + "n": 2, + "positives": 1 + }, + "HLA-B*40:01": { + "auroc": 0.4666666666666667, + "average_precision": 0.7226190476190477, + "brier": 0.4048368493871514, + "n": 8, + "positives": 5 + }, + "HLA-B*44:02": { + "auroc": 0.5125, + "average_precision": 0.6842147435897434, + "brier": 0.5705962359098319, + "n": 13, + "positives": 8 + }, + "HLA-B*55:01": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9723301576064242, + "n": 3, + "positives": 3 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.25555068008958337, + "n": 3, + "positives": 2 + }, + "HLA-C*03:03": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.50740073893866, + "n": 3, + "positives": 2 + }, + "HLA-C*05:01": { + "auroc": 0.0, + "average_precision": 0.6388888888888888, + "brier": 0.6178133595147751, + "n": 4, + "positives": 3 + }, + "HLA-C*06:02": { + "auroc": 0.5, + "average_precision": 0.8875, + "brier": 0.6729075510492147, + "n": 5, + "positives": 4 + }, + "HLA-C*07:01": { + "auroc": 0.32857142857142857, + "average_precision": 0.5365388579674294, + "brier": 0.4838775810121097, + "n": 12, + "positives": 7 + }, + "HLA-C*07:02": { + "auroc": 0.3333333333333333, + "average_precision": 0.8055555555555556, + "brier": 0.38905007015972415, + "n": 4, + "positives": 3 + }, + "HLA-C*12:03": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9803353164996088, + "n": 2, + "positives": 2 + } + }, + "metadata": { + "score_direction": "higher", + "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/bigmhc-v1.0.csv", + "task": "immunogenicity", + "version": "v1.0" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.6518518518518518, + "ndcg@10": 0.6312873089172459, + "ndcg@20": 0.7738604771130371, + "ndcg@5": 0.5325765609825637, + "positive_bearing_patients": 9, + "precision@10": 0.5611111111111111, + "precision@20": 0.5646005983467284, + "precision@5": 0.5333333333333333, + "recall@10": 0.7159050825717492, + "recall@20": 1.0, + "recall@5": 0.35287798621131955 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.5535088711420252, + "low": 0.39938211912672045 + }, + "average_precision": { + "high": 0.673066296054345, + "low": 0.49899434265684595 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 1.0 + }, + "mrr": { + "high": 0.837037037037037, + "low": 0.45365740740740734 + }, + "ndcg@10": { + "high": 0.7275479609299087, + "low": 0.5328394305376711 + }, + "ndcg@20": { + "high": 0.8561682035271525, + "low": 0.6647997182152893 + }, + "ndcg@5": { + "high": 0.6403384470711495, + "low": 0.4175916065340972 + }, + "precision@10": { + "high": 0.6666666666666666, + "low": 0.45 + }, + "precision@20": { + "high": 0.6469078556462458, + "low": 0.4669648367055487 + }, + "precision@5": { + "high": 0.6444444444444445, + "low": 0.4222222222222222 + }, + "recall@10": { + "high": 0.8320025653358986, + "low": 0.6028407086740422 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.4365211640211639, + "low": 0.28138327721661055 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8762995389962468, + "ndcg@20": 0.8762995389962468, + "ndcg@5": 0.7590911424516827, + "precision@10": 0.7, + "precision@20": 0.6363636363636364, + "precision@5": 0.8, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.5714285714285714 + }, + "RCC-102": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.2, + "ndcg@10": 0.430624116386567, + "ndcg@20": 0.430624116386567, + "ndcg@5": 0.23719771276929622, + "precision@10": 0.25, + "precision@20": 0.25, + "precision@5": 0.2, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.5 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.6395647163312487, + "ndcg@20": 0.8837407779997418, + "ndcg@5": 0.7227265726449519, + "precision@10": 0.5, + "precision@20": 0.5294117647058824, + "precision@5": 0.6, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.41666666666666663, + "ndcg@10": 0.5918332819916706, + "ndcg@20": 0.7898052753909603, + "ndcg@5": 0.46905661104071605, + "precision@10": 0.7, + "precision@20": 0.7368421052631579, + "precision@5": 0.6, + "recall@10": 0.5, + "recall@20": 1.0, + "recall@5": 0.21428571428571427 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.47534276898500805, + "ndcg@20": 0.7203074868223488, + "ndcg@5": 0.36005461457723403, + "precision@10": 0.5, + "precision@20": 0.5625, + "precision@5": 0.4, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.2222222222222222 + }, + "RCC-107": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7729548681688856, + "ndcg@20": 0.8573636767513857, + "ndcg@5": 0.5531464700081437, + "precision@10": 0.5, + "precision@20": 0.5454545454545454, + "precision@5": 0.4, + "recall@10": 0.8333333333333334, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.41666666666666663, + "ndcg@10": 0.4628984070136053, + "ndcg@20": 0.6780957041773529, + "ndcg@5": 0.46905661104071605, + "precision@10": 0.4, + "precision@20": 0.4666666666666667, + "precision@5": 0.6, + "recall@10": 0.5714285714285714, + "recall@20": 1.0, + "recall@5": 0.42857142857142855 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6832491119818491, + "ndcg@20": 0.807381612705331, + "ndcg@5": 0.5118886180624318, + "precision@10": 0.8, + "precision@20": 0.6875, + "precision@5": 0.6, + "recall@10": 0.7272727272727273, + "recall@20": 1.0, + "recall@5": 0.2727272727272727 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7488189704001319, + "ndcg@20": 0.9211261047873995, + "ndcg@5": 0.710970696247901, + "precision@10": 0.7, + "precision@20": 0.6666666666666666, + "precision@5": 0.6, + "recall@10": 0.7, + "recall@20": 1.0, + "recall@5": 0.3 + } + }, + "pooled": { + "auroc": 0.4759748427672956, + "average_precision": 0.5856631462955934, + "brier": 0.44860391491063, + "n": 128, + "positives": 75 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.4759748427672956, + "average_precision": 0.5856631462955934, + "brier": 0.44860391491063, + "n": 128, + "positives": 75 + } + } + }, + "DeepHLApan": { + "hla": { + "HLA-A*01:01": { + "auroc": 0.5555555555555556, + "average_precision": 0.9467813051146384, + "brier": 0.6350796566107824, + "n": 10, + "positives": 9 + }, + "HLA-A*02:01": { + "auroc": 0.5277777777777778, + "average_precision": 0.425, + "brier": 0.30187009644184803, + "n": 9, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.8333333333333334, + "average_precision": 0.8333333333333333, + "brier": 0.21866561025884895, + "n": 5, + "positives": 2 + }, + "HLA-A*25:01": { + "auroc": null, + "average_precision": null, + "brier": 0.5919859914807039, + "n": 1, + "positives": 0 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.6518509783239685, + "n": 3, + "positives": 0 + }, + "HLA-A*31:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.22453314818114337, + "n": 2, + "positives": 1 + }, + "HLA-A*32:01": { + "auroc": 0.5, + "average_precision": 0.8333333333333333, + "brier": 0.29385490350386284, + "n": 3, + "positives": 2 + }, + "HLA-A*66:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.2637184827870923, + "n": 4, + "positives": 2 + }, + "HLA-B*07:02": { + "auroc": 0.75, + "average_precision": 0.8333333333333333, + "brier": 0.24624852418541207, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.4666666666666667, + "average_precision": 0.6983333333333334, + "brier": 0.3105990778488774, + "n": 8, + "positives": 5 + }, + "HLA-B*13:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.409583986666841, + "n": 2, + "positives": 1 + }, + "HLA-B*18:01": { + "auroc": 0.25, + "average_precision": 0.5833333333333333, + "brier": 0.30231696703191596, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.6666666666666666, + "average_precision": 0.6011904761904762, + "brier": 0.36036220595666985, + "n": 10, + "positives": 4 + }, + "HLA-B*27:06": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.18737445074360148, + "n": 1, + "positives": 1 + }, + "HLA-B*35:01": { + "auroc": null, + "average_precision": null, + "brier": 0.5986074455543146, + "n": 1, + "positives": 0 + }, + "HLA-B*35:03": { + "auroc": 0.0, + "average_precision": 0.3333333333333333, + "brier": 0.647851957956463, + "n": 3, + "positives": 1 + }, + "HLA-B*38:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.47431751632294605, + "n": 2, + "positives": 1 + }, + "HLA-B*40:01": { + "auroc": 0.2, + "average_precision": 0.5392857142857144, + "brier": 0.503263362761755, + "n": 8, + "positives": 5 + }, + "HLA-B*44:02": { + "auroc": 0.7625, + "average_precision": 0.8607954545454546, + "brier": 0.29576446274958446, + "n": 13, + "positives": 8 + }, + "HLA-B*55:01": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9321453855555113, + "n": 3, + "positives": 3 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.30637106710129497, + "n": 3, + "positives": 2 + }, + "HLA-C*03:03": { + "auroc": 0.0, + "average_precision": 0.5833333333333333, + "brier": 0.3276111573648398, + "n": 3, + "positives": 2 + }, + "HLA-C*05:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.22813751061878165, + "n": 4, + "positives": 3 + }, + "HLA-C*06:02": { + "auroc": 0.5, + "average_precision": 0.8875, + "brier": 0.19676999380006047, + "n": 5, + "positives": 4 + }, + "HLA-C*07:01": { + "auroc": 0.4714285714285714, + "average_precision": 0.6553030303030304, + "brier": 0.2791138986130422, + "n": 12, + "positives": 7 + }, + "HLA-C*07:02": { + "auroc": 0.6666666666666666, + "average_precision": 0.9166666666666666, + "brier": 0.23313610190493347, + "n": 4, + "positives": 3 + }, + "HLA-C*12:03": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.0001787875426514692, + "n": 2, + "positives": 2 + } + }, + "metadata": { + "score_direction": "higher", + "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "task": "immunogenicity", + "version": "1.1.1@ac1f4beb" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 0.8888888888888888, + "mrr": 0.7222222222222222, + "ndcg@10": 0.6703258245072995, + "ndcg@20": 0.8085230459234908, + "ndcg@5": 0.6140248752438171, + "positive_bearing_patients": 9, + "precision@10": 0.5611111111111111, + "precision@20": 0.5646005983467284, + "precision@5": 0.6222222222222222, + "recall@10": 0.7264870931537598, + "recall@20": 1.0, + "recall@5": 0.35645342312008976 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.595005662825475, + "low": 0.40889232966390093 + }, + "average_precision": { + "high": 0.7418858290949979, + "low": 0.5420228402887045 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 0.6666666666666666 + }, + "mrr": { + "high": 0.9074074074074074, + "low": 0.5185185185185185 + }, + "ndcg@10": { + "high": 0.773125749993384, + "low": 0.5659855649509037 + }, + "ndcg@20": { + "high": 0.8947689844911606, + "low": 0.6899714059669765 + }, + "ndcg@5": { + "high": 0.7654367605421801, + "low": 0.4067505026362102 + }, + "precision@10": { + "high": 0.6501388888888882, + "low": 0.4611111111111111 + }, + "precision@20": { + "high": 0.6469078556462458, + "low": 0.4669648367055487 + }, + "precision@5": { + "high": 0.7555555555555556, + "low": 0.4222222222222222 + }, + "recall@10": { + "high": 0.8537918871252205, + "low": 0.601262225428892 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.4670554753888087, + "low": 0.23950296616963282 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.7730836500855317, + "ndcg@20": 0.8497584640784246, + "ndcg@5": 0.6130227926089763, + "precision@10": 0.6, + "precision@20": 0.6363636363636364, + "precision@5": 0.6, + "recall@10": 0.8571428571428571, + "recall@20": 1.0, + "recall@5": 0.42857142857142855 + }, + "RCC-102": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 0.0, + "mrr": 0.16666666666666666, + "ndcg@10": 0.4227898344066503, + "ndcg@20": 0.4227898344066503, + "ndcg@5": 0.0, + "precision@10": 0.25, + "precision@20": 0.25, + "precision@5": 0.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.0 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.6592243328151071, + "ndcg@20": 0.8935911984826842, + "ndcg@5": 0.830419897363192, + "precision@10": 0.5, + "precision@20": 0.5294117647058824, + "precision@5": 0.8, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.4444444444444444 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5683416927717359, + "ndcg@20": 0.8218204494927667, + "ndcg@5": 0.6608397947263839, + "precision@10": 0.6, + "precision@20": 0.7368421052631579, + "precision@5": 0.8, + "recall@10": 0.42857142857142855, + "recall@20": 1.0, + "recall@5": 0.2857142857142857 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5161528906822678, + "ndcg@20": 0.7543146979573652, + "ndcg@5": 0.5296347172140421, + "precision@10": 0.5, + "precision@20": 0.5625, + "precision@5": 0.6, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-107": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.9448192267559941, + "ndcg@20": 0.9448192267559941, + "ndcg@5": 0.8539316501572937, + "precision@10": 0.6, + "precision@20": 0.5454545454545454, + "precision@5": 0.8, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.6666666666666666 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5798612725328838, + "ndcg@20": 0.7275358570132512, + "ndcg@5": 0.38356636737133565, + "precision@10": 0.5, + "precision@20": 0.4666666666666667, + "precision@5": 0.4, + "recall@10": 0.7142857142857143, + "recall@20": 1.0, + "recall@5": 0.2857142857142857 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7914544336296033, + "ndcg@20": 0.9106496109753879, + "ndcg@5": 0.7860137352654724, + "precision@10": 0.8, + "precision@20": 0.6875, + "precision@5": 0.8, + "recall@10": 0.7272727272727273, + "recall@20": 1.0, + "recall@5": 0.36363636363636365 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7772050868859213, + "ndcg@20": 0.9514280741488932, + "ndcg@5": 0.8687949224876582, + "precision@10": 0.7, + "precision@20": 0.6666666666666666, + "precision@5": 0.8, + "recall@10": 0.7, + "recall@20": 1.0, + "recall@5": 0.4 + } + }, + "pooled": { + "auroc": 0.5046540880503144, + "average_precision": 0.6532494332203217, + "brier": 0.361873408840907, + "n": 128, + "positives": 75 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.5046540880503144, + "average_precision": 0.6532494332203217, + "brier": 0.361873408840907, + "n": 128, + "positives": 75 + } + } + }, + "DeepImmuno-CNN": { + "hla": { + "HLA-A*01:01": { + "auroc": null, + "average_precision": 0.9999999999999999, + "brier": 0.3949711436600252, + "n": 6, + "positives": 6 + }, + "HLA-A*02:01": { + "auroc": 0.2777777777777778, + "average_precision": 0.4666666666666667, + "brier": 0.36143655748097425, + "n": 6, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.4841049884591555, + "n": 2, + "positives": 1 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.8053670747968357, + "n": 3, + "positives": 0 + }, + "HLA-B*07:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.11387628173357213, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.75, + "average_precision": 0.6666666666666666, + "brier": 0.3782338557424658, + "n": 4, + "positives": 2 + }, + "HLA-B*18:01": { + "auroc": 0.75, + "average_precision": 0.8333333333333333, + "brier": 0.2261920761561721, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.625, + "average_precision": 0.6499999999999999, + "brier": 0.3437516617313712, + "n": 8, + "positives": 4 + }, + "HLA-B*40:01": { + "auroc": 0.0, + "average_precision": 0.6388888888888888, + "brier": 0.23982545942086936, + "n": 4, + "positives": 3 + }, + "HLA-B*44:02": { + "auroc": 0.6666666666666666, + "average_precision": 0.3333333333333333, + "brier": 0.21491416359774884, + "n": 4, + "positives": 1 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.2952957789199111, + "n": 3, + "positives": 2 + }, + "HLA-C*06:02": { + "auroc": 0.3333333333333333, + "average_precision": 0.8055555555555556, + "brier": 0.25271668358889426, + "n": 4, + "positives": 3 + } + }, + "metadata": { + "score_direction": "higher", + "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deepimmuno-cnn.csv", + "task": "immunogenicity", + "version": "1.0@df42ac5b" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8095238095238095, + "ndcg@10": 0.8191558587095117, + "ndcg@20": 0.8368671618581434, + "ndcg@5": 0.7107525131913099, + "positive_bearing_patients": 7, + "precision@10": 0.616156462585034, + "precision@20": 0.6232993197278911, + "precision@5": 0.6523809523809524, + "recall@10": 0.9642857142857143, + "recall@20": 1.0, + "recall@5": 0.7547619047619047 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.6588901357351014, + "low": 0.2826005353179266 + }, + "average_precision": { + "high": 0.7066785307286951, + "low": 0.5130564168759791 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 1.0 + }, + "mrr": { + "high": 0.9523809523809523, + "low": 0.6388888888888888 + }, + "ndcg@10": { + "high": 0.9222423078369432, + "low": 0.7102434954027206 + }, + "ndcg@20": { + "high": 0.9254379579736113, + "low": 0.7425358171776326 + }, + "ndcg@5": { + "high": 0.8388566073648231, + "low": 0.5911799981092667 + }, + "precision@10": { + "high": 0.7626798115079363, + "low": 0.5107142857142857 + }, + "precision@20": { + "high": 0.7708333333333334, + "low": 0.525 + }, + "precision@5": { + "high": 0.8, + "low": 0.5238095238095238 + }, + "recall@10": { + "high": 1.0, + "low": 0.8928571428571429 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.9111111111111111, + "low": 0.5833333333333334 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8446574402245026, + "ndcg@20": 0.8446574402245026, + "ndcg@5": 0.705601260713727, + "precision@10": 0.6666666666666666, + "precision@20": 0.6666666666666666, + "precision@5": 0.6, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.75 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.8219303906225648, + "ndcg@20": 0.8219303906225648, + "ndcg@5": 0.5087403079104241, + "precision@10": 0.5, + "precision@20": 0.5, + "precision@5": 0.4, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.4 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@5": 1.0, + "precision@10": 1.0, + "precision@20": 1.0, + "precision@5": 1.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 1.0 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6309297535714575, + "ndcg@20": 0.6309297535714575, + "ndcg@5": 0.6309297535714575, + "precision@10": 0.5, + "precision@20": 0.5, + "precision@5": 0.5, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 1.0 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6108096776415199, + "ndcg@20": 0.7347887996819422, + "ndcg@5": 0.5147714448836774, + "precision@10": 0.45, + "precision@20": 0.5, + "precision@5": 0.6, + "recall@10": 0.75, + "recall@20": 1.0, + "recall@5": 0.5 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8627766724467997, + "ndcg@20": 0.8627766724467997, + "ndcg@5": 0.773049051499641, + "precision@10": 0.5714285714285714, + "precision@20": 0.5714285714285714, + "precision@5": 0.6666666666666667, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.8333333333333334 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.962987076459737, + "ndcg@20": 0.962987076459737, + "ndcg@5": 0.8421757737602428, + "precision@10": 0.625, + "precision@20": 0.625, + "precision@5": 0.8, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.8 + } + }, + "pooled": { + "auroc": 0.4717868338557994, + "average_precision": 0.5833356847201822, + "brier": 0.334029412618985, + "n": 51, + "positives": 29 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.4717868338557994, + "average_precision": 0.5833356847201822, + "brier": 0.334029412618985, + "n": 51, + "positives": 29 + } + } + }, + "PRIME": { + "hla": { + "HLA-A*01:01": { + "auroc": 0.8888888888888888, + "average_precision": 0.9888888888888888, + "brier": 0.7091975897099, + "n": 10, + "positives": 9 + }, + "HLA-A*02:01": { + "auroc": 0.8055555555555556, + "average_precision": 0.5888888888888888, + "brier": 0.27258663554044443, + "n": 9, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.8333333333333334, + "average_precision": 0.8333333333333333, + "brier": 0.32875926336680006, + "n": 5, + "positives": 2 + }, + "HLA-A*25:01": { + "auroc": null, + "average_precision": null, + "brier": 0.000241056676, + "n": 1, + "positives": 0 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.029397898114000003, + "n": 3, + "positives": 0 + }, + "HLA-A*31:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.4250756048725, + "n": 2, + "positives": 1 + }, + "HLA-A*32:01": { + "auroc": 0.5, + "average_precision": 0.8333333333333333, + "brier": 0.5460592483973333, + "n": 3, + "positives": 2 + }, + "HLA-A*66:01": { + "auroc": 0.5, + "average_precision": 0.75, + "brier": 0.42045441972875, + "n": 4, + "positives": 2 + }, + "HLA-B*07:02": { + "auroc": 0.5, + "average_precision": 0.75, + "brier": 0.42103460183875, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.4666666666666667, + "average_precision": 0.645, + "brier": 0.48618577261425, + "n": 8, + "positives": 5 + }, + "HLA-B*13:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.299968537172, + "n": 2, + "positives": 1 + }, + "HLA-B*18:01": { + "auroc": 0.25, + "average_precision": 0.5833333333333333, + "brier": 0.5449431063919999, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.7916666666666666, + "average_precision": 0.8125, + "brier": 0.32862863951029997, + "n": 10, + "positives": 4 + }, + "HLA-B*27:06": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9869568108490001, + "n": 1, + "positives": 1 + }, + "HLA-B*35:01": { + "auroc": null, + "average_precision": null, + "brier": 6.0074010000000005e-06, + "n": 1, + "positives": 0 + }, + "HLA-B*35:03": { + "auroc": 0.0, + "average_precision": 0.3333333333333333, + "brier": 0.32795754596533333, + "n": 3, + "positives": 1 + }, + "HLA-B*38:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.341915561845, + "n": 2, + "positives": 1 + }, + "HLA-B*40:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.526717705342375, + "n": 8, + "positives": 5 + }, + "HLA-B*44:02": { + "auroc": 0.3625, + "average_precision": 0.5592147435897437, + "brier": 0.5715760481483078, + "n": 13, + "positives": 8 + }, + "HLA-B*55:01": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9283741232836666, + "n": 3, + "positives": 3 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.4117236931343333, + "n": 3, + "positives": 2 + }, + "HLA-C*03:03": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.6127647742116666, + "n": 3, + "positives": 2 + }, + "HLA-C*05:01": { + "auroc": 0.3333333333333333, + "average_precision": 0.8055555555555556, + "brier": 0.65865841489875, + "n": 4, + "positives": 3 + }, + "HLA-C*06:02": { + "auroc": 0.0, + "average_precision": 0.6791666666666667, + "brier": 0.7228783354532, + "n": 5, + "positives": 4 + }, + "HLA-C*07:01": { + "auroc": 0.44285714285714284, + "average_precision": 0.5720315398886827, + "brier": 0.56084362498675, + "n": 12, + "positives": 7 + }, + "HLA-C*07:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.70853305068425, + "n": 4, + "positives": 3 + }, + "HLA-C*12:03": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.8546234622425, + "n": 2, + "positives": 2 + } + }, + "metadata": { + "score_direction": "higher", + "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/prime-2.0.csv", + "task": "immunogenicity", + "version": "2.0" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.7685185185185185, + "ndcg@10": 0.6881137179921539, + "ndcg@20": 0.8271212105514111, + "ndcg@5": 0.6908650160295152, + "positive_bearing_patients": 9, + "precision@10": 0.5611111111111111, + "precision@20": 0.5646005983467284, + "precision@5": 0.6814814814814815, + "recall@10": 0.7203944203944204, + "recall@20": 1.0, + "recall@5": 0.47526054192720857 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.6460297912268356, + "low": 0.5190087535014005 + }, + "average_precision": { + "high": 0.7184616323154791, + "low": 0.5803605807721791 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 1.0 + }, + "mrr": { + "high": 0.9074074074074074, + "low": 0.6296296296296297 + }, + "ndcg@10": { + "high": 0.7462504477591396, + "low": 0.6222563694605704 + }, + "ndcg@20": { + "high": 0.884401594746618, + "low": 0.7626559123915736 + }, + "ndcg@5": { + "high": 0.789022108580068, + "low": 0.6109516552594605 + }, + "precision@10": { + "high": 0.6444444444444444, + "low": 0.4666666666666667 + }, + "precision@20": { + "high": 0.6469078556462458, + "low": 0.4669648367055487 + }, + "precision@5": { + "high": 0.8, + "low": 0.5703703703703704 + }, + "recall@10": { + "high": 0.821869488536155, + "low": 0.6236159211159211 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.6257114798781466, + "low": 0.3651551226551226 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8047526145467748, + "ndcg@20": 0.8814274285396677, + "ndcg@5": 0.7590911424516827, + "precision@10": 0.6, + "precision@20": 0.6363636363636364, + "precision@5": 0.8, + "recall@10": 0.8571428571428571, + "recall@20": 1.0, + "recall@5": 0.5714285714285714 + }, + "RCC-102": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6240505200038379, + "ndcg@20": 0.6240505200038379, + "ndcg@5": 0.6240505200038379, + "precision@10": 0.25, + "precision@20": 0.25, + "precision@5": 0.4, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 1.0 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5030399532861565, + "ndcg@20": 0.7513611520351536, + "ndcg@5": 0.5147714448836774, + "precision@10": 0.5, + "precision@20": 0.5294117647058824, + "precision@5": 0.6, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7951725670061051, + "ndcg@20": 0.9624379607364066, + "ndcg@5": 1.0, + "precision@10": 0.7, + "precision@20": 0.7368421052631579, + "precision@5": 1.0, + "recall@10": 0.5, + "recall@20": 1.0, + "recall@5": 0.35714285714285715 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6866059341140196, + "ndcg@20": 0.8042717497692193, + "ndcg@5": 0.6608397947263839, + "precision@10": 0.7, + "precision@20": 0.5625, + "precision@5": 0.8, + "recall@10": 0.7777777777777778, + "recall@20": 1.0, + "recall@5": 0.4444444444444444 + }, + "RCC-107": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7891499901405781, + "ndcg@20": 0.8735587987230782, + "ndcg@5": 0.6843515475204855, + "precision@10": 0.5, + "precision@20": 0.5454545454545454, + "precision@5": 0.6, + "recall@10": 0.8333333333333334, + "recall@20": 1.0, + "recall@5": 0.5 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.75, + "ndcg@10": 0.6573429591465089, + "ndcg@20": 0.8003441152775891, + "ndcg@5": 0.5922216874835863, + "precision@10": 0.5, + "precision@20": 0.4666666666666667, + "precision@5": 0.6, + "recall@10": 0.7142857142857143, + "recall@20": 1.0, + "recall@5": 0.42857142857142855 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.6051925781121545, + "ndcg@20": 0.8417183321921483, + "ndcg@5": 0.5742421908816506, + "precision@10": 0.6, + "precision@20": 0.6875, + "precision@5": 0.5333333333333333, + "recall@10": 0.5454545454545454, + "recall@20": 1.0, + "recall@5": 0.2424242424242424 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7277163455732502, + "ndcg@20": 0.9049208376855988, + "ndcg@5": 0.8082168163143322, + "precision@10": 0.7, + "precision@20": 0.6666666666666666, + "precision@5": 0.8, + "recall@10": 0.7, + "recall@20": 1.0, + "recall@5": 0.4 + } + }, + "pooled": { + "auroc": 0.579874213836478, + "average_precision": 0.6393093241046907, + "brier": 0.5017369006240234, + "n": 128, + "positives": 75 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.579874213836478, + "average_precision": 0.6393093241046907, + "brier": 0.5017369006240234, + "n": 128, + "positives": 75 + } + } + } + }, + "paired_same_task": [ + { + "ci_high": 0.04947664122641821, + "ci_low": -0.1161501900458979, + "difference_left_minus_right": -0.028679245283018795, + "left": "BigMHC", + "left_value_common": 0.4759748427672956, + "metric": "auroc", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.5046540880503144, + "task": "immunogenicity" + }, + { + "ci_high": -0.003714798993252979, + "ci_low": -0.12680520185134853, + "difference_left_minus_right": -0.06758628692472834, + "left": "BigMHC", + "left_value_common": 0.5856631462955934, + "metric": "average_precision", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6532494332203217, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.33333333333333337, + "ci_low": 0.0, + "difference_left_minus_right": 0.11111111111111116, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.8888888888888888, + "task": "immunogenicity" + }, + { + "ci_high": 0.001898148148147924, + "ci_low": -0.18148148148148147, + "difference_left_minus_right": -0.07037037037037042, + "left": "BigMHC", + "left_value_common": 0.6518518518518518, + "metric": "mrr", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.7222222222222222, + "task": "immunogenicity" + }, + { + "ci_high": 0.012336267625158463, + "ci_low": -0.08993220825260795, + "difference_left_minus_right": -0.03903851559005356, + "left": "BigMHC", + "left_value_common": 0.6312873089172459, + "metric": "ndcg@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6703258245072995, + "task": "immunogenicity" + }, + { + "ci_high": -0.009628459310849956, + "ci_low": -0.06086790188393626, + "difference_left_minus_right": -0.03466256881045371, + "left": "BigMHC", + "left_value_common": 0.7738604771130371, + "metric": "ndcg@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.8085230459234908, + "task": "immunogenicity" + }, + { + "ci_high": 0.050409043936959864, + "ci_low": -0.1901851493238087, + "difference_left_minus_right": -0.08144831426125343, + "left": "BigMHC", + "left_value_common": 0.5325765609825637, + "metric": "ndcg@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6140248752438171, + "task": "immunogenicity" + }, + { + "ci_high": 0.0444444444444444, + "ci_low": -0.0444444444444444, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5611111111111111, + "metric": "precision@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.5611111111111111, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5646005983467284, + "metric": "precision@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.5646005983467284, + "task": "immunogenicity" + }, + { + "ci_high": 0.06666666666666665, + "ci_low": -0.2222222222222222, + "difference_left_minus_right": -0.0888888888888889, + "left": "BigMHC", + "left_value_common": 0.5333333333333333, + "metric": "precision@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6222222222222222, + "task": "immunogenicity" + }, + { + "ci_high": 0.044973544973545, + "ci_low": -0.06878306878306872, + "difference_left_minus_right": -0.010582010582010581, + "left": "BigMHC", + "left_value_common": 0.7159050825717492, + "metric": "recall@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.7264870931537598, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.15916426166426165, + "ci_low": -0.1300316658649992, + "difference_left_minus_right": -0.003575436908770213, + "left": "BigMHC", + "left_value_common": 0.35287798621131955, + "metric": "recall@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.35645342312008976, + "task": "immunogenicity" + }, + { + "ci_high": 0.16264296120925142, + "ci_low": -0.04712865259740257, + "difference_left_minus_right": 0.07210031347962381, + "left": "BigMHC", + "left_value_common": 0.5438871473354232, + "metric": "auroc", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.4717868338557994, + "task": "immunogenicity" + }, + { + "ci_high": 0.1148432249105829, + "ci_low": -0.03556215089261466, + "difference_left_minus_right": 0.048296319647384345, + "left": "BigMHC", + "left_value_common": 0.6316320043675665, + "metric": "average_precision", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.5833356847201822, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.125, + "ci_low": -0.3214285714285714, + "difference_left_minus_right": -0.04761904761904767, + "left": "BigMHC", + "left_value_common": 0.7619047619047619, + "metric": "mrr", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8095238095238095, + "task": "immunogenicity" + }, + { + "ci_high": 0.05335537459609574, + "ci_low": -0.09325707120581819, + "difference_left_minus_right": -0.010454564627022767, + "left": "BigMHC", + "left_value_common": 0.8087012940824889, + "metric": "ndcg@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8191558587095117, + "task": "immunogenicity" + }, + { + "ci_high": 0.040171356613644446, + "ci_low": -0.0973475628913677, + "difference_left_minus_right": -0.01610746654958295, + "left": "BigMHC", + "left_value_common": 0.8207596953085604, + "metric": "ndcg@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8368671618581434, + "task": "immunogenicity" + }, + { + "ci_high": 0.044278942373923366, + "ci_low": -0.09740725277182755, + "difference_left_minus_right": -0.013826215159676747, + "left": "BigMHC", + "left_value_common": 0.6969262980316332, + "metric": "ndcg@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7107525131913099, + "task": "immunogenicity" + }, + { + "ci_high": 0.022222222222222143, + "ci_low": 0.0, + "difference_left_minus_right": 0.0071428571428571175, + "left": "BigMHC", + "left_value_common": 0.6232993197278911, + "metric": "precision@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.616156462585034, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.6232993197278911, + "metric": "precision@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6232993197278911, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.6523809523809524, + "metric": "precision@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6523809523809524, + "task": "immunogenicity" + }, + { + "ci_high": 0.03703703703703709, + "ci_low": 0.0, + "difference_left_minus_right": 0.011904761904761862, + "left": "BigMHC", + "left_value_common": 0.9761904761904762, + "metric": "recall@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.9642857142857143, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.7547619047619047, + "metric": "recall@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7547619047619047, + "task": "immunogenicity" + }, + { + "ci_high": -0.015272674169330787, + "ci_low": -0.1891589126906106, + "difference_left_minus_right": -0.10389937106918234, + "left": "BigMHC", + "left_value_common": 0.4759748427672956, + "metric": "auroc", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.579874213836478, + "task": "immunogenicity" + }, + { + "ci_high": 0.01740314444245059, + "ci_low": -0.12351507022065718, + "difference_left_minus_right": -0.053646177809097306, + "left": "BigMHC", + "left_value_common": 0.5856631462955934, + "metric": "average_precision", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6393093241046907, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.07407407407407407, + "ci_low": -0.30370370370370364, + "difference_left_minus_right": -0.1166666666666667, + "left": "BigMHC", + "left_value_common": 0.6518518518518518, + "metric": "mrr", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7685185185185185, + "task": "immunogenicity" + }, + { + "ci_high": 0.03017239737900372, + "ci_low": -0.1448315833102618, + "difference_left_minus_right": -0.05682640907490799, + "left": "BigMHC", + "left_value_common": 0.6312873089172459, + "metric": "ndcg@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6881137179921539, + "task": "immunogenicity" + }, + { + "ci_high": 0.008075273494051982, + "ci_low": -0.11794338501591044, + "difference_left_minus_right": -0.05326073343837401, + "left": "BigMHC", + "left_value_common": 0.7738604771130371, + "metric": "ndcg@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.8271212105514111, + "task": "immunogenicity" + }, + { + "ci_high": -0.023271734532436354, + "ci_low": -0.2990950049162747, + "difference_left_minus_right": -0.15828845504695155, + "left": "BigMHC", + "left_value_common": 0.5325765609825637, + "metric": "ndcg@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6908650160295152, + "task": "immunogenicity" + }, + { + "ci_high": 0.06666666666666676, + "ci_low": -0.06666666666666665, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5611111111111111, + "metric": "precision@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5611111111111111, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5646005983467284, + "metric": "precision@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5646005983467284, + "task": "immunogenicity" + }, + { + "ci_high": -0.0444444444444444, + "ci_low": -0.2594444444444445, + "difference_left_minus_right": -0.14814814814814814, + "left": "BigMHC", + "left_value_common": 0.5333333333333333, + "metric": "precision@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6814814814814815, + "task": "immunogenicity" + }, + { + "ci_high": 0.07215007215007219, + "ci_low": -0.08113275613275624, + "difference_left_minus_right": -0.004489337822671136, + "left": "BigMHC", + "left_value_common": 0.7159050825717492, + "metric": "recall@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7203944203944204, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": -0.03366442199775537, + "ci_low": -0.23490259740259736, + "difference_left_minus_right": -0.12238255571588902, + "left": "BigMHC", + "left_value_common": 0.35287798621131955, + "metric": "recall@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.47526054192720857, + "task": "immunogenicity" + }, + { + "ci_high": 0.13381416410650276, + "ci_low": -0.17358745742336204, + "difference_left_minus_right": -0.010971786833855801, + "left": "DeepHLApan", + "left_value_common": 0.4608150470219436, + "metric": "auroc", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.4717868338557994, + "task": "immunogenicity" + }, + { + "ci_high": 0.12016090914540065, + "ci_low": -0.0844896794670285, + "difference_left_minus_right": 0.03270811775896276, + "left": "DeepHLApan", + "left_value_common": 0.6160438024791449, + "metric": "average_precision", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.5833356847201822, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.20020833333333227, + "ci_low": -0.09999999999999998, + "difference_left_minus_right": 0.0357142857142857, + "left": "DeepHLApan", + "left_value_common": 0.8452380952380952, + "metric": "mrr", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8095238095238095, + "task": "immunogenicity" + }, + { + "ci_high": 0.12901205374691374, + "ci_low": -0.04765365130313093, + "difference_left_minus_right": 0.03239542824625208, + "left": "DeepHLApan", + "left_value_common": 0.8515512869557638, + "metric": "ndcg@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8191558587095117, + "task": "immunogenicity" + }, + { + "ci_high": 0.0819166098642451, + "ci_low": -0.05020882164937479, + "difference_left_minus_right": 0.014684125097620404, + "left": "DeepHLApan", + "left_value_common": 0.8515512869557638, + "metric": "ndcg@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8368671618581434, + "task": "immunogenicity" + }, + { + "ci_high": 0.11382296763686184, + "ci_low": -0.0833180287588855, + "difference_left_minus_right": 0.015728646900082577, + "left": "DeepHLApan", + "left_value_common": 0.7264811600913925, + "metric": "ndcg@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7107525131913099, + "task": "immunogenicity" + }, + { + "ci_high": 0.06428571428571428, + "ci_low": 0.0, + "difference_left_minus_right": 0.021428571428571463, + "left": "DeepHLApan", + "left_value_common": 0.6375850340136054, + "metric": "precision@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.616156462585034, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.6232993197278911, + "metric": "precision@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6232993197278911, + "task": "immunogenicity" + }, + { + "ci_high": 0.08571428571428574, + "ci_low": -0.08000000000000007, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.6523809523809524, + "metric": "precision@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6523809523809524, + "task": "immunogenicity" + }, + { + "ci_high": 0.1071428571428571, + "ci_low": 0.0, + "difference_left_minus_right": 0.0357142857142857, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "recall@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.9642857142857143, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.08571428571428574, + "ci_low": -0.08000000000000007, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.7547619047619047, + "metric": "recall@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7547619047619047, + "task": "immunogenicity" + }, + { + "ci_high": 0.04368853820754026, + "ci_low": -0.195250402792696, + "difference_left_minus_right": -0.07522012578616355, + "left": "DeepHLApan", + "left_value_common": 0.5046540880503144, + "metric": "auroc", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.579874213836478, + "task": "immunogenicity" + }, + { + "ci_high": 0.09172009118667658, + "ci_low": -0.09238799817801838, + "difference_left_minus_right": 0.013940109115631039, + "left": "DeepHLApan", + "left_value_common": 0.6532494332203217, + "metric": "average_precision", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6393093241046907, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": -0.33333333333333337, + "difference_left_minus_right": -0.11111111111111116, + "left": "DeepHLApan", + "left_value_common": 0.8888888888888888, + "metric": "hitrate@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.12962962962962965, + "ci_low": -0.2314814814814815, + "difference_left_minus_right": -0.04629629629629628, + "left": "DeepHLApan", + "left_value_common": 0.7222222222222222, + "metric": "mrr", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7685185185185185, + "task": "immunogenicity" + }, + { + "ci_high": 0.08331803966539883, + "ci_low": -0.12058836256824594, + "difference_left_minus_right": -0.017787893484854433, + "left": "DeepHLApan", + "left_value_common": 0.6703258245072995, + "metric": "ndcg@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6881137179921539, + "task": "immunogenicity" + }, + { + "ci_high": 0.04597761031229554, + "ci_low": -0.09102404902752281, + "difference_left_minus_right": -0.0185981646279203, + "left": "DeepHLApan", + "left_value_common": 0.8085230459234908, + "metric": "ndcg@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.8271212105514111, + "task": "immunogenicity" + }, + { + "ci_high": 0.09313139286054935, + "ci_low": -0.26593716874969686, + "difference_left_minus_right": -0.07684014078569812, + "left": "DeepHLApan", + "left_value_common": 0.6140248752438171, + "metric": "ndcg@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6908650160295152, + "task": "immunogenicity" + }, + { + "ci_high": 0.06666666666666676, + "ci_low": -0.06666666666666665, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.5611111111111111, + "metric": "precision@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5611111111111111, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.5646005983467284, + "metric": "precision@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5646005983467284, + "task": "immunogenicity" + }, + { + "ci_high": 0.0888888888888889, + "ci_low": -0.20000000000000007, + "difference_left_minus_right": -0.059259259259259234, + "left": "DeepHLApan", + "left_value_common": 0.6222222222222222, + "metric": "precision@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6814814814814815, + "task": "immunogenicity" + }, + { + "ci_high": 0.07744107744107742, + "ci_low": -0.07319223985890644, + "difference_left_minus_right": 0.006092672759339446, + "left": "DeepHLApan", + "left_value_common": 0.7264870931537598, + "metric": "recall@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7203944203944204, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.048110068943402265, + "ci_low": -0.36333172999839664, + "difference_left_minus_right": -0.1188071188071188, + "left": "DeepHLApan", + "left_value_common": 0.35645342312008976, + "metric": "recall@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.47526054192720857, + "task": "immunogenicity" + }, + { + "ci_high": 0.04963112522686011, + "ci_low": -0.3485843515037594, + "difference_left_minus_right": -0.15047021943573663, + "left": "DeepImmuno-CNN", + "left_value_common": 0.4717868338557994, + "metric": "auroc", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.622257053291536, + "task": "immunogenicity" + }, + { + "ci_high": 0.03802709044411525, + "ci_low": -0.1543269255374634, + "difference_left_minus_right": -0.04858767874061665, + "left": "DeepImmuno-CNN", + "left_value_common": 0.5833356847201822, + "metric": "average_precision", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6319233634607988, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.20000000000000007, + "ci_low": -0.09999999999999998, + "difference_left_minus_right": 0.0357142857142857, + "left": "DeepImmuno-CNN", + "left_value_common": 0.8095238095238095, + "metric": "mrr", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.7738095238095238, + "task": "immunogenicity" + }, + { + "ci_high": 0.05146739445932063, + "ci_low": -0.0835917238927503, + "difference_left_minus_right": -0.007970910076125937, + "left": "DeepImmuno-CNN", + "left_value_common": 0.8191558587095117, + "metric": "ndcg@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.8271267687856376, + "task": "immunogenicity" + }, + { + "ci_high": 0.05207977361323347, + "ci_low": -0.03209450981710734, + "difference_left_minus_right": 0.009740393072505737, + "left": "DeepImmuno-CNN", + "left_value_common": 0.8368671618581434, + "metric": "ndcg@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.8271267687856376, + "task": "immunogenicity" + }, + { + "ci_high": 0.0826002139424956, + "ci_low": -0.08016590563417962, + "difference_left_minus_right": 0.0036628776135311236, + "left": "DeepImmuno-CNN", + "left_value_common": 0.7107525131913099, + "metric": "ndcg@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.7070896355777788, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": -0.06428571428571428, + "difference_left_minus_right": -0.021428571428571463, + "left": "DeepImmuno-CNN", + "left_value_common": 0.616156462585034, + "metric": "precision@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6375850340136054, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 0.6232993197278911, + "metric": "precision@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6232993197278911, + "task": "immunogenicity" + }, + { + "ci_high": 0.09333333333333338, + "ci_low": -0.11428571428571432, + "difference_left_minus_right": -0.00952380952380949, + "left": "DeepImmuno-CNN", + "left_value_common": 0.6523809523809524, + "metric": "precision@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6619047619047619, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": -0.1071428571428571, + "difference_left_minus_right": -0.0357142857142857, + "left": "DeepImmuno-CNN", + "left_value_common": 0.9642857142857143, + "metric": "recall@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.10000000000000009, + "ci_low": -0.12857142857142856, + "difference_left_minus_right": -0.011904761904761862, + "left": "DeepImmuno-CNN", + "left_value_common": 0.7547619047619047, + "metric": "recall@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.7666666666666666, + "task": "immunogenicity" + } + ] +} diff --git a/results/analysis/rcc/missingness.csv b/results/analysis/rcc/missingness.csv new file mode 100644 index 0000000..2383eb5 --- /dev/null +++ b/results/analysis/rcc/missingness.csv @@ -0,0 +1,10 @@ +predictor,status,count +BigMHC,predicted,128 +BigMHC,unsupported_hla,1 +DeepHLApan,predicted,128 +DeepHLApan,unsupported_hla,1 +DeepImmuno-CNN,predicted,51 +DeepImmuno-CNN,unsupported_hla,29 +DeepImmuno-CNN,unsupported_length,49 +PRIME,predicted,128 +PRIME,unsupported_hla,1 diff --git a/results/analysis/rcc/paired_differences.csv b/results/analysis/rcc/paired_differences.csv new file mode 100644 index 0000000..b8c0ecf --- /dev/null +++ b/results/analysis/rcc/paired_differences.csv @@ -0,0 +1,91 @@ +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,DeepHLApan,immunogenicity,auroc,128,75,9,0.4759748427672956,0.5046540880503144,-0.028679245283018795,-0.1161501900458979,0.04947664122641821 +BigMHC,DeepHLApan,immunogenicity,average_precision,128,75,9,0.5856631462955934,0.6532494332203217,-0.06758628692472834,-0.12680520185134853,-0.003714798993252979 +BigMHC,DeepHLApan,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,hitrate@5,128,75,9,1.0,0.8888888888888888,0.11111111111111116,0.0,0.33333333333333337 +BigMHC,DeepHLApan,immunogenicity,mrr,128,75,9,0.6518518518518518,0.7222222222222222,-0.07037037037037042,-0.18148148148148147,0.001898148148147924 +BigMHC,DeepHLApan,immunogenicity,ndcg@10,128,75,9,0.6312873089172459,0.6703258245072995,-0.03903851559005356,-0.08993220825260795,0.012336267625158463 +BigMHC,DeepHLApan,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.8085230459234908,-0.03466256881045371,-0.06086790188393626,-0.009628459310849956 +BigMHC,DeepHLApan,immunogenicity,ndcg@5,128,75,9,0.5325765609825637,0.6140248752438171,-0.08144831426125343,-0.1901851493238087,0.050409043936959864 +BigMHC,DeepHLApan,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.0444444444444444,0.0444444444444444 +BigMHC,DeepHLApan,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,precision@5,128,75,9,0.5333333333333333,0.6222222222222222,-0.0888888888888889,-0.2222222222222222,0.06666666666666665 +BigMHC,DeepHLApan,immunogenicity,recall@10,128,75,9,0.7159050825717492,0.7264870931537598,-0.010582010582010581,-0.06878306878306872,0.044973544973545 +BigMHC,DeepHLApan,immunogenicity,recall@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,recall@5,128,75,9,0.35287798621131955,0.35645342312008976,-0.003575436908770213,-0.1300316658649992,0.15916426166426165 +BigMHC,DeepImmuno-CNN,immunogenicity,auroc,51,29,9,0.5438871473354232,0.4717868338557994,0.07210031347962381,-0.04712865259740257,0.16264296120925142 +BigMHC,DeepImmuno-CNN,immunogenicity,average_precision,51,29,9,0.6316320043675665,0.5833356847201822,0.048296319647384345,-0.03556215089261466,0.1148432249105829 +BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@10,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,mrr,51,29,9,0.7619047619047619,0.8095238095238095,-0.04761904761904767,-0.3214285714285714,0.125 +BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@10,51,29,9,0.8087012940824889,0.8191558587095117,-0.010454564627022767,-0.09325707120581819,0.05335537459609574 +BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8207596953085604,0.8368671618581434,-0.01610746654958295,-0.0973475628913677,0.040171356613644446 +BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@5,51,29,9,0.6969262980316332,0.7107525131913099,-0.013826215159676747,-0.09740725277182755,0.044278942373923366 +BigMHC,DeepImmuno-CNN,immunogenicity,precision@10,51,29,9,0.6232993197278911,0.616156462585034,0.0071428571428571175,0.0,0.022222222222222143 +BigMHC,DeepImmuno-CNN,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,precision@5,51,29,9,0.6523809523809524,0.6523809523809524,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,recall@10,51,29,9,0.9761904761904762,0.9642857142857143,0.011904761904761862,0.0,0.03703703703703709 +BigMHC,DeepImmuno-CNN,immunogenicity,recall@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,recall@5,51,29,9,0.7547619047619047,0.7547619047619047,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,auroc,128,75,9,0.4759748427672956,0.579874213836478,-0.10389937106918234,-0.1891589126906106,-0.015272674169330787 +BigMHC,PRIME,immunogenicity,average_precision,128,75,9,0.5856631462955934,0.6393093241046907,-0.053646177809097306,-0.12351507022065718,0.01740314444245059 +BigMHC,PRIME,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,hitrate@5,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,mrr,128,75,9,0.6518518518518518,0.7685185185185185,-0.1166666666666667,-0.30370370370370364,0.07407407407407407 +BigMHC,PRIME,immunogenicity,ndcg@10,128,75,9,0.6312873089172459,0.6881137179921539,-0.05682640907490799,-0.1448315833102618,0.03017239737900372 +BigMHC,PRIME,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.8271212105514111,-0.05326073343837401,-0.11794338501591044,0.008075273494051982 +BigMHC,PRIME,immunogenicity,ndcg@5,128,75,9,0.5325765609825637,0.6908650160295152,-0.15828845504695155,-0.2990950049162747,-0.023271734532436354 +BigMHC,PRIME,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.06666666666666665,0.06666666666666676 +BigMHC,PRIME,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,precision@5,128,75,9,0.5333333333333333,0.6814814814814815,-0.14814814814814814,-0.2594444444444445,-0.0444444444444444 +BigMHC,PRIME,immunogenicity,recall@10,128,75,9,0.7159050825717492,0.7203944203944204,-0.004489337822671136,-0.08113275613275624,0.07215007215007219 +BigMHC,PRIME,immunogenicity,recall@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,recall@5,128,75,9,0.35287798621131955,0.47526054192720857,-0.12238255571588902,-0.23490259740259736,-0.03366442199775537 +DeepHLApan,DeepImmuno-CNN,immunogenicity,auroc,51,29,9,0.4608150470219436,0.4717868338557994,-0.010971786833855801,-0.17358745742336204,0.13381416410650276 +DeepHLApan,DeepImmuno-CNN,immunogenicity,average_precision,51,29,9,0.6160438024791449,0.5833356847201822,0.03270811775896276,-0.0844896794670285,0.12016090914540065 +DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@10,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,mrr,51,29,9,0.8452380952380952,0.8095238095238095,0.0357142857142857,-0.09999999999999998,0.20020833333333227 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@10,51,29,9,0.8515512869557638,0.8191558587095117,0.03239542824625208,-0.04765365130313093,0.12901205374691374 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8515512869557638,0.8368671618581434,0.014684125097620404,-0.05020882164937479,0.0819166098642451 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@5,51,29,9,0.7264811600913925,0.7107525131913099,0.015728646900082577,-0.0833180287588855,0.11382296763686184 +DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@10,51,29,9,0.6375850340136054,0.616156462585034,0.021428571428571463,0.0,0.06428571428571428 +DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@5,51,29,9,0.6523809523809524,0.6523809523809524,0.0,-0.08000000000000007,0.08571428571428574 +DeepHLApan,DeepImmuno-CNN,immunogenicity,recall@10,51,29,9,1.0,0.9642857142857143,0.0357142857142857,0.0,0.1071428571428571 +DeepHLApan,DeepImmuno-CNN,immunogenicity,recall@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,recall@5,51,29,9,0.7547619047619047,0.7547619047619047,0.0,-0.08000000000000007,0.08571428571428574 +DeepHLApan,PRIME,immunogenicity,auroc,128,75,9,0.5046540880503144,0.579874213836478,-0.07522012578616355,-0.195250402792696,0.04368853820754026 +DeepHLApan,PRIME,immunogenicity,average_precision,128,75,9,0.6532494332203217,0.6393093241046907,0.013940109115631039,-0.09238799817801838,0.09172009118667658 +DeepHLApan,PRIME,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,hitrate@5,128,75,9,0.8888888888888888,1.0,-0.11111111111111116,-0.33333333333333337,0.0 +DeepHLApan,PRIME,immunogenicity,mrr,128,75,9,0.7222222222222222,0.7685185185185185,-0.04629629629629628,-0.2314814814814815,0.12962962962962965 +DeepHLApan,PRIME,immunogenicity,ndcg@10,128,75,9,0.6703258245072995,0.6881137179921539,-0.017787893484854433,-0.12058836256824594,0.08331803966539883 +DeepHLApan,PRIME,immunogenicity,ndcg@20,128,75,9,0.8085230459234908,0.8271212105514111,-0.0185981646279203,-0.09102404902752281,0.04597761031229554 +DeepHLApan,PRIME,immunogenicity,ndcg@5,128,75,9,0.6140248752438171,0.6908650160295152,-0.07684014078569812,-0.26593716874969686,0.09313139286054935 +DeepHLApan,PRIME,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.06666666666666665,0.06666666666666676 +DeepHLApan,PRIME,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,precision@5,128,75,9,0.6222222222222222,0.6814814814814815,-0.059259259259259234,-0.20000000000000007,0.0888888888888889 +DeepHLApan,PRIME,immunogenicity,recall@10,128,75,9,0.7264870931537598,0.7203944203944204,0.006092672759339446,-0.07319223985890644,0.07744107744107742 +DeepHLApan,PRIME,immunogenicity,recall@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,recall@5,128,75,9,0.35645342312008976,0.47526054192720857,-0.1188071188071188,-0.36333172999839664,0.048110068943402265 +DeepImmuno-CNN,PRIME,immunogenicity,auroc,51,29,9,0.4717868338557994,0.622257053291536,-0.15047021943573663,-0.3485843515037594,0.04963112522686011 +DeepImmuno-CNN,PRIME,immunogenicity,average_precision,51,29,9,0.5833356847201822,0.6319233634607988,-0.04858767874061665,-0.1543269255374634,0.03802709044411525 +DeepImmuno-CNN,PRIME,immunogenicity,hitrate@10,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,mrr,51,29,9,0.8095238095238095,0.7738095238095238,0.0357142857142857,-0.09999999999999998,0.20000000000000007 +DeepImmuno-CNN,PRIME,immunogenicity,ndcg@10,51,29,9,0.8191558587095117,0.8271267687856376,-0.007970910076125937,-0.0835917238927503,0.05146739445932063 +DeepImmuno-CNN,PRIME,immunogenicity,ndcg@20,51,29,9,0.8368671618581434,0.8271267687856376,0.009740393072505737,-0.03209450981710734,0.05207977361323347 +DeepImmuno-CNN,PRIME,immunogenicity,ndcg@5,51,29,9,0.7107525131913099,0.7070896355777788,0.0036628776135311236,-0.08016590563417962,0.0826002139424956 +DeepImmuno-CNN,PRIME,immunogenicity,precision@10,51,29,9,0.616156462585034,0.6375850340136054,-0.021428571428571463,-0.06428571428571428,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,precision@5,51,29,9,0.6523809523809524,0.6619047619047619,-0.00952380952380949,-0.11428571428571432,0.09333333333333338 +DeepImmuno-CNN,PRIME,immunogenicity,recall@10,51,29,9,0.9642857142857143,1.0,-0.0357142857142857,-0.1071428571428571,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,recall@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,recall@5,51,29,9,0.7547619047619047,0.7666666666666666,-0.011904761904761862,-0.12857142857142856,0.10000000000000009 diff --git a/results/analysis/stability/analysis_metadata.json b/results/analysis/stability/analysis_metadata.json new file mode 100644 index 0000000..bad202a --- /dev/null +++ b/results/analysis/stability/analysis_metadata.json @@ -0,0 +1,68 @@ +{ + "analysis_type": "exploratory_descriptive_heterogeneity", + "bootstrap": 2000, + "seed": 20260820, + "ks": [ + 1, + 5, + 10, + 20 + ], + "coverage_thresholds": [ + 0.5, + 0.8, + 0.95 + ], + "datasets": [ + "improve_benchmark", + "zhao_vaccine_benchmark", + "rcc_vaccine_benchmark" + ], + "predictors": { + "BigMHC": { + "task": "immunogenicity", + "version": "v1.0", + "status": { + "predicted": 128, + "unsupported_hla": 1 + } + }, + "MHCflurry": { + "task": "presentation", + "version": "2.2.1", + "status": { + "predicted": 2315 + } + }, + "PRIME": { + "task": "immunogenicity", + "version": "2.0", + "status": { + "predicted": 128, + "unsupported_hla": 1 + } + }, + "DeepHLApan": { + "task": "immunogenicity", + "version": "1.1.1@ac1f4beb", + "status": { + "predicted": 128, + "unsupported_hla": 1 + } + }, + "DeepImmuno-CNN": { + "task": "immunogenicity", + "version": "1.0@df42ac5b", + "status": { + "predicted": 51, + "unsupported_length": 49, + "unsupported_hla": 29 + } + } + }, + "limitations": [ + "fixed pretrained scores; no causal inference", + "model-selection probabilities are conditional on observed patient samples", + "coverage strategies are descriptive and not multiplicity-adjusted" + ] +} diff --git a/results/analysis/stability/dataset_predictor_metric_matrix.csv b/results/analysis/stability/dataset_predictor_metric_matrix.csv new file mode 100644 index 0000000..d12f058 --- /dev/null +++ b/results/analysis/stability/dataset_predictor_metric_matrix.csv @@ -0,0 +1,61 @@ +coverage,dataset,metric,n,predictor,value +1.0,improve_benchmark,AUROC,17475,BigMHC,0.5458288293413741 +1.0,improve_benchmark,Recall@1,17475,BigMHC,0.01890873015873016 +1.0,improve_benchmark,Recall@5,17475,BigMHC,0.038045634920634924 +1.0,improve_benchmark,Recall@10,17475,BigMHC,0.07884220626867686 +1.0,improve_benchmark,Recall@20,17475,BigMHC,0.14580332700248363 +1.0,improve_benchmark,AUROC,17475,MHCflurry,0.5367554822274058 +1.0,improve_benchmark,Recall@1,17475,MHCflurry,0.009533730158730158 +1.0,improve_benchmark,Recall@5,17475,MHCflurry,0.04661971434997751 +1.0,improve_benchmark,Recall@10,17475,MHCflurry,0.08175583503756877 +1.0,improve_benchmark,Recall@20,17475,MHCflurry,0.20210686751655038 +1.0,improve_benchmark,AUROC,17475,PRIME,0.5969085863470571 +1.0,improve_benchmark,Recall@1,17475,PRIME,0.008503455608718766 +1.0,improve_benchmark,Recall@5,17475,PRIME,0.0589177645722564 +1.0,improve_benchmark,Recall@10,17475,PRIME,0.13437829088804587 +1.0,improve_benchmark,Recall@20,17475,PRIME,0.26004746077835117 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,BigMHC,0.5423340961098398 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,BigMHC,0.39791666666666664 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,BigMHC,0.9541666666666666 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,BigMHC,0.9875 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,BigMHC,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,DeepHLApan,0.5072058035931643 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,DeepHLApan,0.35625 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,DeepHLApan,0.9249999999999998 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,DeepHLApan,1.0 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,DeepHLApan,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,DeepImmuno-CNN,0.5257315351282925 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,DeepImmuno-CNN,0.3770833333333333 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,DeepImmuno-CNN,0.9249999999999998 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,DeepImmuno-CNN,0.9875 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,DeepImmuno-CNN,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,MHCflurry,0.5430968726163234 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,MHCflurry,0.3572916666666667 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,MHCflurry,0.9322916666666666 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,MHCflurry,1.0 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,MHCflurry,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,PRIME,0.5365240112306668 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,PRIME,0.37083333333333335 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,PRIME,0.9302083333333334 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,PRIME,1.0 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,PRIME,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,BigMHC,0.5438871473354232 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,BigMHC,0.15952380952380954 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,BigMHC,0.7785714285714286 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,BigMHC,0.9761904761904762 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,BigMHC,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,DeepHLApan,0.4608150470219436 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,DeepHLApan,0.1880952380952381 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,DeepHLApan,0.7785714285714285 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,DeepHLApan,1.0 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,DeepHLApan,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,DeepImmuno-CNN,0.4717868338557994 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,DeepImmuno-CNN,0.16428571428571428 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,DeepImmuno-CNN,0.7785714285714286 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,DeepImmuno-CNN,0.9761904761904762 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,DeepImmuno-CNN,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,PRIME,0.622257053291536 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,PRIME,0.15952380952380954 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,PRIME,0.7785714285714285 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,PRIME,1.0 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,PRIME,1.0 diff --git a/results/analysis/stability/endpoint_domain_auroc.png b/results/analysis/stability/endpoint_domain_auroc.png new file mode 100644 index 0000000000000000000000000000000000000000..75d60ee1400a003ca19cc83d4abb2fc790c155d6 GIT binary patch literal 91312 zcmdRWgP^^n8O+{xL>-cE>5K!8txhr`mv z#lcyUpWpW1XYkoOS@1jcP_DtN&^lb#b4H;~bz3QbBdmx;TgX5aQ;g4q@zY;$EKAGlXmqMDDLfXMs#}40$O>Xmk+wX1B z-MAUQxvSnkWf!XMhQHyqxINbp_Fzmy9UU4UFCiAwh{n^xmG>U9k-emY`~UMl0Vv`B z{KQcmLP7tZpRY6SPlmU&_n>|J-=`=r1|W9>d7P)&*YfvIsDS+nDCYnC9Kt~(Px+sp zkL;(g{Lhc_k2z5PKfUlVjMP(mIV;vh@)`{HS9pgCm@0GY<)@yfdbh$Baw_SZ|G`4i6S=vv#s1b5m654# zJi1&f?PjQ;c}H%~`r_C~)$Wea(CUDHm044K_l4%;AzhsKZkr-`dsv^jM8A?whSL5^ zchZAWRipUE89cW)mPd*Q{KA$d+Uu7mNt6{5G#uL{8_jaN(JJ>!93Irh2xSl!cQr(9w_&cwvMzAz%BbWvJmC$h`<=t~Nm)NIAt@7D{{#RIYPcQQ0? zBuaa}S4+C`;IYeWe_s&4e!=xv5zJ`V*k9MDe!|}3cG>f+-whWmL-#qX^jR^x-dt+E6pPCeDUaH$ zJ3qzDGmBXNd`5rzx62{x>mc3tFuu8rZqqLtF|82u%@ZU5m}-# z620nJ!bR1p@(A-fEw@k7)I+L zyD^@^va73>C@oR93p)v8_f3ONc&@@{-N~vw{f+ecaG1cp@*K<0BxCQ!-n_Z>u_OYG z?B&KM#4iPvkp-zwsX}DZx`)px2Zg-9_PQ=vA#fO$-=)lUgVTLyYn>}O-JHUOWscgn z!s=G>&4^11{xZfT4(~iW*e#jG?52jYl{k!#${S83C~Zh~DhM?k{n`*C)F`6*+;x)3 z`BM^Qa*mQ26>vUTH}7Gn2u2&*W39{;qHEdxQo?yO+L8C*I|VKJ@}0SF3@ZkPf%7co z-m5t*a(;$}{_@nEPFy6h6Am z_S@#aTP~5F|9;di%73hhHB7*kl{C)OtlzIRQ8YRI;{$q8exhEHg&v?t3 z?n>X!@e`ABjrYpCsb3vrICH3n=B&86*7~t_?#Q zoa8s=nI;!U30t*&AEKeS5k^!usq(8}A_iC_v2kIh>+q%N*!knqICUphWcJ2H z@Q5Gs7+hyOWJ(5sjgNtN`vZ&x93!D=GKQ@z<%V-d~|72ytF>G zXncC>=mm}N;o82jpu0HZzWq6;g)Q&zYmW8ZZr7BPs5sy;-tul`-fQ_!OcRT#L1o&l zcZzMr_9pWp_BjmtROef8ssv{fp&HlC<#OzFFLrvX-87j*#fyLc$lu|*)2z})M;eDL}Xb*V^QR=iDO^!{R+|jznPB=G@F-c$cQbK{*0HvOZdx7 z_=s-^E{Wuf1vbq#7HSTpELGHPywfc{BTB5C7_+g(jp6ZQH&%@X+Fi&zJt|M`N!I_K z67q8KdLCB!d}2LL%H0lobA2M=wxqZ>0}=C9qkT-WX!a=0ZBY~|()?(~$%V&Cc z^bb#o1S$6-{tWwpN04LnW)RypwNsLnzzf*VobA-R;$+>28ap!n7M< zHL@lP!?m_hTF7@5Gr5tkm;Y zl_bb*h$e|UekEobh#91-#Z1+=O&3DvBZ%5`T`}k@bG4mlOM9KK!TQ&+i}xqnGqBTe zjx@vu0wM+}9@Eb(APWOly_*Bf-oL;ewmU7S8c8!+t zc3Uhi_gw6OwsdmRX-UM4_X6~-6j!ACt$12@d^kMoJXboFh%eq=8^R_OtuHpqP1}ZY zs-lfC{g$Mh1m7J3yT5eJ5){5`5)7-;WQ)ZMCWh?$w#&+$MqWvX)x&`pitpU^n#?Y? zImu@@w|2^s9A)gqUALa06mjXE&Fd4q3zl(yJL?t(aEBT&HL)fpi*OF|5+q&g3TmN} z1Yz_PjXwLjX1|)2`ErsVSV(SgCy5F#YVgbGD14 zjY7D?1lKGVvwwFn4M))O}x|sSX{y6h~|J`kYQEtE?w%=bI zE`qjL(u+VZ7UEe??qSI@Yyp*?i|N^lPybpmpP=@zF!ozF_l|>A zB5b7nnyuO`qW;T>9xnPy2ekqFr6BG4+p|Q5-^^yDFXlGnECqsLsnd_>D!1vLl@(m& z#wQi_9ajne?d2#nF< zJu|fZlceYJm3+(fMLXk%$?H0~_Z7$h>>kQcZBPctmgxsmaa3-v7!;FJDeT?J@5!IN zSM6~7TIMU285>6|LwUevbd*yP})6s@W$UG}o&`s+&!iSXhnzDBT}^WkXLPe)O0 z=Q|Aeu6B>L%D<;7P4ImgE$8nSUuo$FwLu^<;+)DM1HgC*eqUfk-LuG3(baD+XtLpP z00c#1#F{E?{zM&4(8S*rJ~bMxkRE9F+)u!1>FYuMXmfpq+uMq%yI5JZcyWQj&B?U` z+LXLA1y=1ferkrkj`PpZ^Lmp+e(zu?6!(*eWv-LS4D!wK67BsOLU@fN+35l!*yz*K z&knI#FA|}ishvnIq-zX!S^9odvn#{uO3#3gt1r#>NiD4T5}k<5!0Y3e(U*5KF3^kTaKkgP6@fV>A_;dOy%d|b2 zZGxKff+m-{9v*lvTrcu2DdmX=!$m-K6%|yQOXAA@?CtFF%fS~WcfH=UlFBQ`B+$e^ zgO@fXD`(?&eAGSabv93Ak*GA+7npxHjvd>6Hv$;)RlcDNHhU<&{>6@@>&m%Yw)$~` zF7}OVRGHjHuf99=-Arv>V+zY{y6F-+Ov{*Yv7U((6lpi+0OIFrmtEFoPZ{M_oSvrK zwoAC{pRq$Se!-LN&F8%MqAuO5n~E9|?z5;EA{e21=1V3eJ}TH_wk?_ym**k^Zz%#r z3%FV*ur@L=ICJosABX!1v;NQSxKiKNdk#I_QB)X@;kPPkHP@DS6pbo}qK;a`=+9JI z8dv%C!5I+U6kMX*1gQSLpmo}=r@$(F+A-4im~Whj&|Umj!`7^GjnCTYlE#_RW8ajT z)rMYVuQM`Vx4hLZnVv@rJwLk}s)z72g7uv?h1-8vXDDcxw2;lSZ9CeIwdy@Ox*h5U z=rw>kIc1loz-?Op=oPQg7>oBl>#k8xF%?`cE$qI(6|1i6*XP=uTp3vy`ldso_OIew z7PX>R)Fr1XOplKxMaX2tOwQ~JXvJ2CY06sVGqPqImbPhsxOJwuyI2QjTxh|kPZ0DX z7Vg}-Q|e@n?Wgov?p<5ZAF;|6s3f$JC-a6T&-(9dp(_(*yt&4m!$W2%^qYp}eto;$ zPgsidA_Tla8zL>A`v3mHX>IMc~b zCM(#|zw+Mb{_EYFzt266vZYnXv&?PUS~mXA$-b^SXYtLpWG2Sgw-Z6=d*bMy;s>9V zZoZeW>lNQDFJsYRy0$TbyAj5q2B%S_u;2}4XyplGrSl_&b#eDieS*ymDO)y_!ak!y_~CSVF}z6RoVYHe#)>n^ zd2ARvVYz)h`_bnoK6^zrO1k@R7`8Dp0jE*m`^E$1fq{7QDap2-lkt5r&NICd-c-3e zAEtA(moT`B6F5`>6@hg&y46{VubftX61zm)vHD_QqLd@Pf=?zhZ3n9m;3@eP)0z9; zHGZD4ra;Nxx}=>IeK%WQgjjtK`zsP?p{_MC%c|yrM*Kwv*|lVwfy%zEDga>%!(p0U zG*g@Dj|~l4wvU@pZoN#NvTts#}JTwJeGk;@Vhy~P!E1}EXjYyMb-l8RX5(A zeX2utTK;pdyVyQy3&2jk=OS9jl3_-Bb_m)&_PY+p>gq_8i758%+qZ&ty(OpG&z(JM zwzW2oE?0{Y)T;9HeTj<^vZ#X!+Tij05Nmu zA3S)lGXL3qw#Lec==i!)+JtvN=eM`-+}Na3>-%df^IYn&KcPM}B#)IJ@(MNx+%dw% z{ec~k54|y}o>kVTWLt#A2tFsLX-yD%3-pfH0)6wnZtgs;OvuI_NQ`?@ru~CUmLQnp zw=r>HG5U-WmFH9*;mDH2&7Z3%ouxuMPhd)EjzcvXzEgVs-Y||8*9;J*cw6EY@P9mZcb>q0 zgpJLvCdqS@>7!w3e*JW-Y5JItVYVK=#xJ-d$H<_}eU^{5(Q!8x7@M(Djx^lag)fv0 z(T;shRXZE?=46co$sZdP`y9dr>vn-&y-#IdAs1p!&W@zsjJWByHLvnfEA3UhoPWPh zhz|2DCV_`f2=%S}Y)zFRQchcflYG~Q8lcL=``vI$_n7~!_319RP&3@}VGx-#s>(4T zEq}5682$2y65-Nyzkj?Q8E;LoBlnlNs=|_H#cZo{5vaIzby~NzFB0`Jv9U}MOcYLe6yox>n?809jevF zU^6L=N`U7Lv&e$Au`hl2WgWhn0lY;sALq?*2Ps@-%`&w-z2!9v)}!`Vb458ZTF2~H z>ONR=A2_kGFLq3^I5@H5IbOX5*_VMn7GX&rZhxDWJb04NhMin>Je;?0HO7hv z$;ImulqpA=Ic;|tf?R6RjGQG$<5(n|N*eqe8)EY#Jtx{Y zaft^m4{-v1Ur9Ex**xZJY(bv7v9btSo4WD{058++_g3@g&++K4k_5y%X{wy%)r{`WL^f3!s^BqsMcIFEu;F;2KWuU|P@_=|^4fyL%@;$=WXEm+&Xn2AwbwrTppSTp-%d18U%k9nbhAqFwp z+Ii+%$?_CQMp28ywL^i~N*B+7I)u9V&g7Fw7~w7YBq9W{v2$zM&64;vR%iNzhVp#% zUN%|dCR^#+y9Dp{MR4YN90}?qTY`{cT@}%)pNM{C^(@BkU4GHEsu%j*9}>}hROB7~ zrN|EGkFoq3(Er{@Z!fiv&U$^$j5q)bD2Clvm=_~#73%A=yR)qY)fGDm+-X*gmdOIg zxgBaNbCQ5%kK)nG|6W!O;=t7yIX>ZloUu2u_OP9V6|8v8Q_{raTeFNYvFNsm?4^Eb zO>O1$<2J!NoTYG#S4Q~ZSK|YN6}bH8JD00XaP{HbyR(d7sxXc%iqYMt|&yt`l4Y!))Az>eJkq`ccyV- z{sR&2y?5APHqxOMsdw9e(7q@&v2+xt@V(r3mcK_z*xo10Z!Tocj%$KF@3 zJxi=&dv~-k4rJWJVWpWORM|Ybns&}~E8DK+^u)SXCWY1!+isoqRolcYO;UTN4$cFo zg3YE19a0Po38v6h2`(10?=5Lf8L{okt(Ij=Vt&r-)5842i<}cHVESx({iTNec46hM zNlaZFyYEt$M898Faf_ruWrVBS+@JzfkKp}}aXAb-UPk(Q#$^oy{=0qV%9VHp%ox`b z;B=loJj<~U4tO8gD5YI0tTS1n=3H)3VkqskGm%w!xn4r`$5-g?D(|}-`s#)C=6h5* zJEtZZ3q3*35eSZ1PEoXxwi>!w>9amHRQbVmhL&JD^=I+NuB*l;RZ=uh5wX$z**ji(w?zK(>y_FG$6RhLx-8+%P~dKbM$$5}rMy0@ZO<+1r& zTWfl%u`EqnC`Nihcd>GF=5-<6bE=opGJeGA){QNF>1_I$SAM0wK&Gc-OaiIsoq9Cv z-}wIdm@&0W>iLoA7P4G(+P;f2cO%wkxl<>>Z@x3@0b%Se{;BV6Eg*$uvTH-GA7dtn z&SSNc1om#*V8Zt})0mcK^(%&%zrV&!=Iz}2?6quNJR;_^&B93>`ZKxOc>ie7W@HHI z+5NSxv8f)Lgp-NrNR{S&!F~X-wK8&b6-1Y=vCfh4h4}6Y z?_yr}nckHmzz5Eyjo>&G{YZcC?b(XCEdV7e(6?r~5ZS^$;~TWbhXokW2YolX@fLkK zVz)L=8$q3GEQgI(AK1qJ)ytCy=Zh5C<3oSAKM+85PVaLa4 zrYrATq_n3Kz5!@%&uDYfY|(a_YLpFve6qp6n3vrBq@8Wu^GAX;VJ7YZ!3m_XBG7-j z$;6L$E+{0OHZjiZt)O08(Dm|`+1c5gowi-}as3h1e4C@6l87^zonP*nmzjm;M7y$A z)Gz$gFPHn`OmF8sytbL|wK8S(8C2Z*qK#5*+yK1U;-}A_lXr854*Yd2mwybb88^u) z0x)@_v;m7W1U=Zb{+g(QVHyf{aH1!vZW})waP%$(r zdcT7A&R8cM-DOiH)dP1w+^U|WJJQEjF>NgLy1-eiamuFk{nOfoj^IgQZv*=u#gdz? z8uoApxaV)D-8}AXt2e!Efak|_e0X3jW9@4KFILZMxqZL!g_Ao51Bh)n%EV7#$V<(Y zyriu%Su(j80%k(M3=Xd`FbZA`akU47i>aD$&S-Y_Ns#r`_uF395y~ORn}{X0n5Gg; zfy0i9q~Ulkle@HlcjBde?ZPV)m%HRPr|?=ih7vbGKAm`S-9C!QbBT~O=-%&1$UG$8 z=gN~-Ok*$>lc-7DQO2FlZced_lk{9j_1poi%_g7MwTviCJVXsn-DQxFgl$<&Qb0j- zDrMC_tuIs#^vX)mtB~tsVqnB&9Z|yVv0}S!63@-Xx%TDpNn?zuBsz=`fx|J-u9w}_ ze|25ecYN_Z)1ap8dTMQXyXlAaf+qY-^RI8u8znyBu5Bamf%DyXB{*BEu*dA>A^{T# z6t5c)v{c*1oozceR#NeWYqDb1PT32QVkFbd4s27(Sc+U?%6BhA-|b2!o5zkwGaUT* zWM}JS9{|@hIMitNfcBkXL*HAr0Qu)R$bVPmvprzgVpk|Z?wyj%eKsA;-QfV z*iAMFye88z5qLN)z5#t>pWAj-UTt(V0_ak({coL?{>@QbH71N zr1r1iW5u6h#k0yVEin(yocb=N^~YIf=$t zkvnaYqmgL0nC^-;K)fJBt|WUgP2q!9qjJ@L)22V zRLD&Joa_h49mvzPXFVTF^ln;e%6Z1XK2y-n)96ObiU3D3b{E6-Bgubf!mS&w_!flW z(s<)aAJ01~?RiQk>@B&9(ROe$q=<=4B0^aTfmBCKVsCr~X}dO7B(q*Lb{TJ0dn7FF zPjC@zCxctFxJ;Wns=+K0?>mkKGZ11!_{--W>nvKY+yR<2m2k;wRL5n*XM9p!XrF1l zCGimDvf+#D;Iv}lb2o%ycyg>2F3a#QujhctmsQ_M8d72RJr5$e`(}?zM}gIyJM~(4 z8JUS%R)HLK?uW|se&Sk|3fGcI%CCT=v-&a-gV{HMETZARV#QIT_LtZH0%(NL%r zf&tS-36&%(6>@A0+qSQ!P_>8>IgkC>xNwT@0QFMN$F(-8_XWiKexqN(emlgVjd7RF z!U4+ZAkrOZ?Shk?t3vKccMxe|k!4?{GVLK0>Iip5gt63v#YorFd2Rd}UQ+M)G9b1v zdg7|P!tw3gtVeA0hLg5CUawO!8S*#IXvD&697Nphn6l)X!^ zIsMYXEHM*5!&cI>O5-IDD+f*c`sHVA_S;$_(ES(vEADCAD+Fth-W5mxVYujZFrVsE z;nMT|{2Y@x;V&s^$Cpl7>m9Ifs z=rlktFQ=Px2><*v7E1NO^+o;3hkbs&tQbZ`QVPzZ!gdRcn^BR}v2u79(b6kjrfdg3 z^*Q=X|5)7^#FETi%69cbuSt1yIBR1`;Bcy~Q&vz}=Bi-BB2^p;{p_3MDn#@roTp#8 zc@>!w4z*LnQk5=g`&Xc?Q8xqUh@Qzf&WdaUTxQ<<59guRn0q#}6qT}(iH6whh12@| zU%N%P-a+i7lG^h0B$A3Li?A$pf0Z&EB#)ONCQMRx_wtw ze`|$ok2b8oK9NTnv9$QAFIsyu)uFJjtYSk#2OAfe(sF2X3I8ba2og{zEh4NK{#mY$ zoteQuVWZwYql&v+)~COtnsm^2wzK)qM(?<&l%Jh@YEr}!b29DOjdgidYduyT$<29% zfo}zd )y8e&XkDJLfH1@>;7!$)?tp?sDEeS?>D^*7U-4-dXEi=~?GWP#)mpXIj1 z_U|0_rb1)kP`1XN&W#rv*OZe?w;xjNXU5g9{py)!qW@z_ua`}NZaM71=E^ZK$h?n3 zG<*<#J3vKPY7E~ji;t?{SPQ!UW%9^QGE;;$H99eT0RDQHol_kCGZ-G})spuu)`|`jO}HczHswq01GWV-v1)tr zpXauel~&I59zg}%Qb5iS%0o?x(ApGP(9X=EJ68sI9vWjaBpQ|UQnh)6#u$*EP;opa zR{~IJy2?tW?`EuH0Li?qy6`lL#E7EhMlxHR(Xn=3vw0!^U{|;zbDP zZ1tc{`?@t8ViD&6!d1aN=DIp##BErTTVT_z#pW}okQlXJ`JZ%Fz<^PO*F6ZwaDt*2 zIWCd^p9IgknGAqY5msx=SWj!AEm=EV1L|OV`f_a<l{QSj{)<=W4)m6#eJhPNlW@=|C;`9VNhvK91Rv3VI36b|8$HJgMN zul|jaRn>z9KGj=#=Vlb2?d-6=s^ePS%cB>}<_4?L<dbi;ALu}IHQMLy~Toh@boIZW>=^%tJ zP-~Klq8)Mu)ZYY3fVYJ=9Wts7=MMAT>M6v~RvNu~LHAEw?#4|Ij}l*L^P41z5@0?> z*nv{#3)RJ!kG(}AdPgD@hm@(jj-oD}cIQMXoYNZ=qNjMl;`?j)s{5(0ntfyUHt%{e zP^mw2wgC;@g1rhs|8Bvo4CtRBXOLGp{7;Hc@zxVwtwlwuXD+tY`@d92T<{^_5k^Jp zz$zM2s4_u-nJTtD$obE04`4j@0PiVsUK9wJ2o{75C*~)z{yXIju_E>TqI!ZGCD&>G z``qv@RR<(o1hx_S86W_O8W;LY2IeA??lU}6Zd19MpFQT^OD}iaLFDioaiZA>qrj2b zZjL`;+*=tGjly9C#=C67mLX8qAyS|oCmKV5xK7-c{51FzK@dwWE+QTmn+M3MjzQ)b z*WW6lo8q~VlaZo=y$RyL$RldDD->hE7VF=f7E>cGKA}CDD*tF7ZAsMME(>xg#>bD( zWO@RRU6E`E7FOyO$OPtC@2KyWb~9oaG{R6I7TsAY1lGWwB%g(s!jOdsHek^FH&4mP z2)`Q!5cqo!pFxq0z>YS!&w2%xZ<+XYa^mYfRU>)0A&?LT@c$>&it`Y4a1r+aqxOgM zVN)3>oKLD{C|JnO#T~t=%zU@OFp7fM z-c1mlIe~&}L^&dd%XuuA>4?*&pn2VSAm>rJxmTj5`HjlXAHS>}4#g~XSLf5+8weYV zs`uSlA2T2R`PQ!Z*Cehg@{H26=y|}_(WXPM*aCr@*22P=!!@0NvEPW~i5YH;(?Mz? zn5qTRb6+UgX629--F;PX_^$HTbC6r5VL}vexb-FRmF}~~dgUH@v~7HP`B&?r_}O!( z+g0diz~%2Z--McZjrIcu)z#+&0m&niqOhmFRnm*bO+I!2aXb=Mt`5Q;rfO0y;~`l_ zpS6|WFq`k&81UQ7u4O?g1f$_k>t}Me9#f(ioj~Hdn4yu7&w!+IvJ8qfHdbft1^q-8 zi(wZZVPax3?MDJso_~nharenO5dT~h2lie(4f^(dwr$qlZ4hTiK4v5eR*%66uQ#x- z;JmDp6TZE<8mEBIg~Nx`_z3g*$e5wMY%YyD(x4QBsgw(YotKhvk#AbGSY|6XbC3`$ z7y)M?4*If<4-^~PO5bdQ;z;lbYheM0!KTnE_n1FrD||o4oL_h?tJ=YKmLWD`8*oM3 zJY*|=A^}Oge6yEr%EiUdECd|}^=^0jJ&rH%5g8TObt7y2rE@DP1K+%RL;l4 zzll&vI@P%uvL9uQNV?7Ra>Fr-g8Gc~*}kUP@uw$^uVC+_tG%6@o4eQ!$zW?iY@y)` zx9D+x=dq?jQaqca%LgQ_OY|%dQt7jYOQNdvD0|{4UK~VD>5HFmZ6$Azzvy_s-zYh-PkK!;LW zry;Z*iQ+R79D*B!q++2hD3$A{yA>5b{PCBNtm~h95(p8kw&fjr$I1yGsoNzO$(;P%=?446YajwqS|`Y>xq*8*{VUInILJ-u8}Cs4Gkd#F=O^86`~y>{xb|JhU-4;6y)kM(RP#m%tG}s-kw-oUeIPZ1n}k0^iaeLxD&Q)p z$AEkW8IUq~=7Mcc0elFyO;Cf3xebc4YBhkGedOlPg8&qhs7*AXzN6_t#OJbB$wo4d zNoK<{s$?V}jUX~TW#Fcqm7CKxKcfXw>erzR*s<{&mtUlyA6B12Hz69|NwOWnWtq;x3>{Zz?Fp8Jl}#@hT)8%Q6eZqu>CmMvd8B38tv zuiv&Bd(|I#5xGH(K4;ID>QFMCEO&wpmk!4L=xr&wj@XvO-2A}8pr@B16Y}QHAhGUo zktE|%(V~J$-o8c1M?^rvkJ{|?-dj5K&xe08`u*k6Q2~Hg=((%V4I77`8ygnA#IzIv zW(qZ}uW&TLcuJ<^1<8IV3YRXfu{V zLX0h{!LmC)%@fXcnoU>knVKcoWwk%wo*Qz>5*hbi8o!OVMLaT)L=Oy0)I)5&9BRtS zeXk^0-(ISBU{^ell#LfOf5l#W1i4DaS)1B&X6P+Yw7=|Hiz9x<{!o6CN}J#tIO7Uw zSp;`P09=d-GHJX;bYePu_%ON$3BWq-0+bbJ$MDX-OW2RF)Z}JT00m_#953`1#MHv| z3$2kv-xQ(YlE?xC#G_F=K}_rH1`#y{5fyH5*F!~Z^}QWH!Cnz`pD`>2+x$y@^pL91S64Za zqh1{!Ztoqs!OwLeX96W6W7lmEaL-Lb)|q3iC68n45uir-pQWyokw|n|*7A#PU~{a| z7Gi!w*l`$I;7ZBXvui?r;7i(ju5?>)LVZNXF#GTUut1ClL7>M+eY8PGkNkcKXq2GO zngfz8SehLe=%^wFz&RqR&*@RN(lZN(K*^5@Jp(oJ;1U-y=izX+ZLhq%HWO99w-3d! zKVHhc$@s*#w^Xqr_p=_gcUgYXACSbG*Q4_jjao&SeTe4z-A#zD_ngh^=Y9uQedzfK zc1cY`wMDwfpatly!jr99tgU+Woxyb38vBh}!7B)l+?soaemhxtZGhh-&yVqj$+zD|w`2t4V@Y9h8Q)+0Wz3Uubtuh8}-=-VI+Ec{`LFL$cZN#uIAlHvK$EIZ`(9&1HzFBBS-j~RU8 z^z%<6f3>JRyexDd-hcW~Ws=`U?N&CVIapwt2Zf?I$z5B4Y(B&yWk*hD5SvU7-6eo- zP!ynBqqNvnkS(E{DhFV_2Xs{>y9g&0-(lHB^9P#gJBPI z(38`J6p>e_p?VwIfYfn8s7P~%r^Rp8V{|!1kuU(_yH;CfhB53t?*szB5OfzJ4_-`j z-U0Yi#)7jKgHdw-&fq*rzJA)>`s}a&5&=f0mO1MG!pK(zoKg2vSWBNjf36A(7$a>h)iXt{mxu-Vw+~ga-{%&!bHkMo5J;C$gp6-+BXLKpLUbR9Bt{ zzdxLXw%i9wctd(=C_jB!V}PndfT5|uUkrt92RQGMW8|gITvXg=Lb(A8r3JbsxINf) ze;5ei1oQ#^2xpA6XMRJRm?q-!J^ta?3Y9QM()H8!ZkZ1&sB(Nr(1Qux5PhaFVFxg)_S0HBZ+?W7O>4w5HmHj z(zmJ}(<8|r9^>*ZGo>VHPp!(e-v@C~RhF%17o)*I(#yU72z(5MP|nS&?TfdP1AIw`@XDuJ3~oXL5sSY zI({-m;*fmEbE=8~cbhqTO&U_anor-7;kp!IXlTz{YPPB~cD*o%LuEcw85u z@4(l??*Ao+#~!GTb`D~|^c2J*GPvH3q7>s@C}bg!$M7h(vCUM#z@z}k23bwC_n;(5 z8_P=n&jx*zMUf3e$MW&_D1H+pFehHK1azp8?h}*_jd~l)2|CvI)Vb~H>X*Tf5*j+n zr4|B2OniFKVQuagS{itI`ZvY@=VtVTdW)>-15}xQGy>%BI;7Oi0QYfS_Jm4=1n+vt zt`PRC*+Ry^ASM&5AayKg23oo>w&(CWsuT#5@B^e3VQO6*YZlY%jP=d-TAksI5j0;| zmsLE^N&QdBlR5l*GYf71hDoT#>Bt=)>t_|fh2mdNmFm3fg1Dw)2aKJ`j zZb)o7A^Q+@iZ=vX%^*kb*iT*ZKX-!mw)Qsz00dlLwuZ2Bb?yVc7ogq%&v4k2^DJ8x zjeX{?LH~Wz?kR!*3Ok`hSzoUV+l`g!%Loeq`K%6@$_R{8q`QEsC;^+{(gJX#I#aAn zWb)>zKcU?=q6W!g`=GapYEz%}aiMsu)XJ|+4N}sUX5HyW3>*}U5_8&BR~J1yH*ETR z@2KELPn+24?=?}R?;sD5aG3UUiia%yw|U&U${0Q<|GyH#+oIt|{rjJw{#!==dZt11 z_J4kdQ6%}-9=iVfHLp?$Z#`1v-@5`_W4{7o7$XnH#~|kHAbdjcJB0bnpb6dHK|FQD zN`(Qt8-JbPxhUZ@9CAy73b_krwTA8`hhY*Ci61b53Yi5JNVuu~RU_0)Gsp&_LANZ| zf@3bue-OEV^tJ!!8#IR@ULHn4NmXK-qA*Ti^Hu}TZg0pj)raouw{M&v$Jqyhw-|9D z(c^27|HecHG}t6@{>!4Kgy0A;o?_;qgVe%0{4SJ6UV3hS0^?WqHMfJPGarylZBQTZ zx_aQY<{%+LUlfQDwT)#eXaqq?O_}dVa3eabHtYg<*MDV74-pwa!wvnm$Ep-S`14Ob zstAJCUZzQd724k!s9KT0=zjtoe!$>kz5Wu+sJnc|<)_9Q)!VeQ^*N~-*up_70Z{s8 zoeVghvO*7eUyA2Z(~YR_2N0d}94IskAQZ&?0;xO`?5dyE4J8lp;5~?EN`G6R|&os3?5NA+mZC>XC ztW5=c%(I|kIK}w-&GyTz!!c-FL*_LBZi<@AgKWYFqcv9u$-o9iCEmP2e_@yL>ImO% z$yx&+h48_hm0VV6T$E#gFBU>KTm&*2icH$gzu3FKCm+|%Q6n2uz>=c?NE3Ps(j~$p z67exGBt*L(BVPh;A(D9IzV|7-7midORrZkJ?1$SaYDVbn+VThAs5}QBQ6r!w-L88rqk1mU}{5z#Wiiib)rZ3oLI3O+b?p<~#=}K`sb=ip8`uUnR5xB0yQbC23b(9l1ZwbUW^_M|F!Q!l#pY%dnt9i80rmyTCqUj@oC&WtPROn-^ z<;4&KHFp~#QzgFW!kWT+)a35uYCZpmfpKWG6?c9j?(x*Oo$XC?pl^A6N6C z0ucC^o?kcd$%Wq$buM6VuS`MD{IY)f>cVh3OJEx+X0rv2c_Rv(B1`%Yk`>NU}5Ktiq(ERwR>}t+^o~i310m!)f?ic3%HyW+c?V5 z4wH^V8&g$e(zD1j_CR);Xn|cplSOMjvwqFhBsrPUrr;FxJ4{W%@in)2cW5E$EYBnN z#mIOU)w^PdOFAaL$44Zfvo&=&_Q@_MhVW z@R4HfNyFo!{3MYV1|M#{yVza;$u~b&WQyTD)BwJyb||(NDekN4>P91&gX^dvzQU-L zV-eO_eH0(r)G{=^){7A>y{ugtGJX>*p!~5yx>x91a!V~rd{IcF;c)n;?hl3%SMKSb z?0Rp`93J^n-0}C5+Tfzck-=Ezmb);0D;$U(udy&z=~=&zpj$gKeo>y>rME-=pz;2mbmAX>2pNO|wb zHTP7xeU|dx&_~c-pgF6(JjDB1;KA4Zi@oDc0K}M{r|g~fJWx&kMtLD_l4GwTtr1<2 zj!E$9%}Ab!0(Cgk2zE=C{tkd$jVcGvV@SXMB(Hvl#iD+}kSu)&_OzIWvFAtxl%hbT zMnU2i>nkjp5g;Q&ps}Q4vHIbUS21+9nwH7>ly$K&elXdN=TX@>S3OuAXb$0Ew8LE( zh|VZ1a*gro75?lCND%FiN`ayG0|TNb#TM*)REYT7PC9>tk)Kh-98$rRW_9Nya~Qyq zV)E%?-)=#g`3BrXQ=>#81CjX~wBAriTgItm^Je*zv`z!=6YFkgc32rHC%Rz<@2Ot1 zWTJ&E90vwcVW|Ki1s)eP1uNLFa#(jC&djC?>f^mE%G^NDz!3G~1&t6^tPlqQ<9mKX z(3Rfg+3Qmq;EDc_xzQ`OOE4^PP{;>mIoKpqJKJz(43@%fU+adnrFIe8YSPd8$~Gdm z0Mm_~?%;7&(SP34_!2Dx0VjhgrXi_t2^Cce&BKSqv8$D*yc>4#U!<# zXM5(ls2EeC<;vmez<+#g@E-q?yMN|aY1Qs25 zS`$Eki7S${9Jqdi^=m71r)0!hFjG*uwBG2&6FUV`1*XoH%7NdN)0aHbs zf`VFzf|MUC69M@@PZ%9RR~6D+LluXn$7d1#`{?&L;`ag+i-cgLSA}817&uB1 zQ2VaKH)E8?u7bN44irhX`tboyz!9N$|A%8<*`MqWqxM!1@V$EV>UEF}8$ICUs>{2> z?rFoB$RUa=GF0zqOaMc21W2^#5c5>F4lWT1c$g_*E-gaWoEaHF#IdcfX~gvG3f z62NIv>BH761!#IxaBt`AeJpU(T@C~TPP&eGi*tJTkSJaQF0e84NLOc|XQ)MAQ2-8b z3H-27fU({c!N8T`2JuA8oxwl`wJVb5bsx^OLN3)n5V}l;Z_A(~z-)7nYtg4gPWSXs zyZ3v~%IJZU)SaVa?0GFTA4v1xcNB$sR2#5@o+||ERts-Zozbl>ocs2^V&I2SNFvtk z80Jb(-TjpsP_0t^cX!a`po=XKA@fWln+Z4hChXUt0*11LB;S=7&yI~Y{X1SDq9W0l z43Q6{Y=a93Sw_6^DyxaN03U8@{Tmaa={#m>PsJVwki`@BZhANjC)O+hk3=k6rd`P2 zsi{2eZ;_t^q|B)vD=cW)!g1pwLV{rZdiTG>vPhhTuRsuF>SXS+1Fez!G9u)*w+<>X z$e39hXM(&|f6~O~%njCmvPGaNj5D_YofXBxmpARo&NuULK;Y#?bsR`v#el{orkn#8@N>bho_w z?^kDGts2qp1w=<1;nUrr@ZA+r)4~pzRvt@DtI}WqsnL$~RG2H0u;<&#TrqcaHeiw< zJ&OQxiRZo=j^gE&MU73r6Z zKrku}yud+5orq+t;sCLr{DidS+f;NX*54 z{v+ISnprIgCXJ9;ORaZ`2ky)$sRo5#ysZcmsb?YS5Ds#euW5(Fj$d`vSrq?9+~%5fi_%=7=3`r^V~mt&}##@{BfNO zIim9)kz~o<YYy9q%7sW} zR%MmFhUWgp7>vyKa3?oqvA*`JV6V9FOPuJoo(`*ZaDzH!|20(=UXa1Uo2V zWke!8L6Lu8pjnrBU8mj#%%VD^y8{U#LYsg6&2Y>UOhR-8|B2v032y6;)OpY}Sebx{ z06_O|6UNwYy^b?;zw>AzgMQ6r3oBr}K0p)$$53^u}A@(XwGu}Dok6nH!K@h(r$;$F2^vVcSR6*$EH#S+h~Dd4Pu!I{!G zJ$yb$_g!#{}7CDw?$hah~L6L$g$DYLNp(x5=- zA>GB^-ojcUVPey!dM9rx()N78f9;2;=^!=&!j0atnrtWNu=hxwR2uq4xr*3Pz)|CM z*f3!%K6(eb{jDEAevI+|hKyxCl35)HU1~|2#T#yudHqVyrn(J5dI)JQ!Lt>dyAzRv zVS6N{-L^omiDKZrH>E7l%T+FD<=UxKg~{xNPmxcQ$G^Bf>wR}VO2XfN|NR-)Y~QU| zQA)yOm{dfMSn5LZs(h=SvbE!|(wRC?8HS+5?%EV?ca~Xpdu1*efRZ2i%ha1xBYI2C zr;DAA9sVQoO20oviGM{7jnJB17b^~7a@#r>nPS!^o@F)QGvFzd$kA|*=64)uG3{XX zc!5r^cP!W}~_DT&a!hbDyuFqzk~$waVTe*A2vF4eF$ah>Y@Z*%=ax z&>2THQI0`Tr5$3J&%~o`zfSU&lh@v95Y;*ipv1aSBgFdcy`&9cIv+gYJnS>g#NmMS zpfqUHA-h*I8Ylsj=&jEB3)d_W9q=a$7JVKy4zY6{+XS_GE)*gJnH^|Ni^~O`aW>sl zy%ahzGFb8JgV2i9M=jEwNM(HRvwIx%Yl6ahpHX(MVxvXwW^g4Qk$esaeha=?Vz(u7 zpghPK0aNe&t1J8mLK=S8HEDDVj#QRxz{AOqERbS=X!;2^joq^f0tPY++5}duQD=VM zXWTknuS0mO3O-F*@V7hf_Xa#CB^0K6F`n=N4@jY>>~S<$Wj?6d6uuN+1_8|n(L>gP z?bAo(lw=TmEX)0!U2B_DI?9k%1dSS=KhgyOsOS*UK|GbZ+2lER_bG^onVR6;jO(Jr z>itnud|PuJ9+fNaVfxhSXr;h9zt1&GS;&=d0CbdOxJRkA4J7OV^5#oKO>|MdUxcPH zdsvot&?6EUUO6UYd+=4>XyL~^yjOg0!v1yZ*6A3)lKmei&5l8-fxH=QwaJTw%w_FZi2`Mx<0CBtjw~JbBTO zbr$0%EBm&;@`Zk2V#-gk9q&~>cS}{|Y?`kdhC(@70U6`)(b0mEZ5QE9qWUXA!K9o% z2+*_#|8zFn%81l>v-jArHD{9loU${{bx_?;N6YP z#wLNQoVbe~?+qXX2i{jck7$NP?y~Uslj@i++vtrH)#uEcbNS!OcB!!0PI#Hlx1mNK zcB!umCy|9jeAkDuJqOWq9wm%*d*5kj%A@4GHwk?a8t6pg)^m^al3x5_$-cYs?4~OB z+t))>Z$(f~9q!`+1YT;@*p;w#J}1A>=vCcDukxnLiS&EKo(~&hD#^bi&f@lL+z_ze zNYh1B3QA_sptDIahSroX;vQ6dYtK8ZVeOF?sbD*oRBTb;O2a=oB!8>P!I8MX#FjUQ zXVdFsn6<6b3F?rJrkS$HpW#2LtbckSL6tasa5VR|YmK(tK($(m(b3aW^{ZCMd2ZN@ zszN+1YLfQR!Zh3Z{G|&^;aeW`qyZ(R@3GJCAN>H~0V6Yk-kc=<93x>rn^AM8#dz(! zY=YX88eO3b2g}u?)F!Qi&KV%m4jRLRjR|^(NVDpY(wNK$8jMI_7H7$i*}cb;+rmO%VfxlH~>Px@*5~h>_+PRrrk~T^NZ4Q@U5v00JHAabJpy7pY zX-Sh#oSRPNyQTtrG#SL$(^I$;y>KPNZE!Uekq9a*&mge*wj>5N2W^h$5r%)~)rAqk z1cgLV5pC~Bu^+6}YQat*E+e0jSOfNV*uACm>Mjc?C)fb1tgdWvu>TQ9Kxe~+2J>GN zJKMmG*I|pq3~*RMXEyKLNK|8vxtM^W&gux_dFJt?jtQ-@kRErsc{^v%6T|da`iE#V zt~fU~By)I)UP~A^J`t}!XKZjS|4H)==!nmmAHC&>tevtrA041ATxd3*<7INFMrsN7JWDpW$}!&pz(5oe%I{HnXzc_1NIzM zbm=!KcqSu@Ayk=PV~woK1Z4u}%-p>i?V}h+>ug-w@eFm5STd#%ZWF7gHh*NU9KFc7 zp&E?3>HGTDgXzXC9D*UW2>9V@B+yg%E;_7FbIxcS@uSk$2|bTzQbR#4#$*T(PNdL4 z`Ep%O#N?dr^J|x0t)U$8(C^z>?DZkl^c~u|D@AE<)B0(23HWA@5n|r;Xf3rEGDzxAk{3T*c>Wn}m|! z1QQHU&ty19c;%ZjQ%w%89kjPzDXozzegnu^Mdl=z{ZWzl3-AVb7m&&h^e-b_Mdk=1N6PC|Z3x|%H-8uGH}G(04`TP8m>QdaQNC(jPy9@BQ}+d(96h^+1&h%7t{BML^jqEiB>FCR3e{_*ZxcX|611~pX!*$FJ`sbT8gdAnx2T@lxV?9B$8zp$_+N?$W9 z%JAQ%(mDlRa?G97q=CT~m=yJ@+75S}q3xj-OVpH~eZBlksXM{R9X;cF?dt1%UqT(R zrwz8D1&$UOY_E?nFFn{{F4-Qm2q&|amm~G&mT5nz&h@%l?w%H-k=MWFiaNKVVaqD! zBgIp>YU74y)OphQQ$=8Wn(B=K@q5PuwWs*m|Gqhrw;glfeT5U~%AcIwQuu1AI&eNg zkyyAp>T*x?E1SO0yajdQoA*-l?NI_qW|*U#8utFGUZ1g`&hYap_KUqHj=k2O)qGny z;*K4|xb|po)&w8>MU!jW=>`v&6kKu{K$^^}xABUWjJJnGHyOz6}pyAd1ND z?zLg|+`}y?a)d&j#L**PZflreq*^!_Rkm~d8t5qjW+-^y(5Yv1Uxz*=uYC3c>J|}i zhrb@LjcOV;NAQRrtm$~vY2U}qevC)V*bgxDKapnj4)tC#;hE!z&%vuU5S}qQXRF!S zZ+<@$Wr25z>r^13UZ>tYv`43lTJ;H=ZNx)b4K)?(H&PowX_Iwu6=AIr`y6K_QNwud zM&ihzYwJ*7OU1K`qA_Lu%DiD8Z&51Yvu?WWgam@8xcmI`hxQF!H+6M&Jefl%sg896WK!m%6nzWC_=5O@cN#(%SIi z6dDC1ZPjd^u~N;co@5e~rdQJ#hK+r$>f>oo9Y9g(!z`P#N1KVF@Ra@zs=z%YL>;am z91KvLbW-Oqo=_^iix{W?6#p}t2ccF&vrd>B1afBP2Q@#^e_ItOvc0U6fXVg)(E&q; ztIm|Urn~1etm1lzYVgEB?kSViGB7IU-?)FX;j5eQHr96pvZnvxb>tDj;0W~zPP%5# zJ|f#Ol3Hr*X^!F&D=c|lGI)3aR=(O4TR&+%#60gB8KP}Ge4W{*`0oF{3YTpMayeMf z5j0mudW`$*_K&kQU_r`%)3*D#Tmg(Vj%L}9D20^!%)QbcV~B0;{tmRJzl2=CUGKrr zcAwij*kY9;zA9ZlW726XcZ>5{gXw$Jvho1HqCn}E4s3=p<9bR0=Z@xhIpPf zgOy+LY#*KZM#eW+Fr^{B3uH_tVEL>%;qh86WY2S3wr)1;azVZ)622J}teDahoQ;Pa zoYNg(!HBsTS84*>oa;_qCD;}rBTR%?$v1jV`hJYRl}|5P6LaJP>(-x~rcKFA`A<~`_C`?okBL0r5AR6#K7R;^g1S=Z`HtSx ze{6D(S#WY6#ER_inhzVfUM!dG7drJEhmttO?w8Rtq)by(oOM2kXI$gK}$a#)SAagEv>SUHxjN0Hxx zRIwRN=Ka;8N>HQ?@G7U=>E=vrTt!2|MzWc;dQfG2`M1wD92UJ!-4FIF-8JWY*T5T9 z$4jbk<-S`ej+2{@cnxnb3~w!4rb?zrV0r{5Z>4^3LuxZuC2^p5GyrTTh+`=AIQ;6qb$lc|Z; zoAl;B6_KWPsl8*wmnx;5szVitPm_M4?<|+(W>y=X32;Z}*|27<>l8dvDl_BeCwr1V zovgm{kKB6R18Zg+&WG=(DOilkJz=fPyMfCw@6xlGTw|y7e2aJ6wJU8$ALQTD=UIIS z1*Po68|z%@k8sc!@@68)xHI4%4)7IGtj0ceEDvYNRiJQoME-{L++{9qngFn|9k+1w zxfv(~0%^}}X+RAB;Ag#Z`ETx_pC`pC*GS=n#atm#%B|ceUNIpT9nxwtP!i?&8vtP> zV$zU!gupkW_$noS+p--nOH&)OcPz)r9Mx%rdZF5-20<*kR_s}RRNi84ZNrxbv& zS=~XS}mE^BQYV;CX8-{wOT#PWFHcj{{Lxi3uP_H76}UPJ^!vz zeOC)`rjCPr%Lg@^&?2mnulh)*{`)B?1=WOHNML;sjmwZ9eTT-j|Hd|g185yXnY17V zBr;mGmfKn)6^Wnx(jTBxxwv)o>|)aR;bFgh|Ct3HKkP@7XE%10$au~tYO14L*#)5f z5u$|xAo1$vG^>HLuj1C2QZh7L((RmX#?giZQBQP(3BI7$Bl{p+8u6PvT87U1nO`UW@39$mIp-L|9AXH zp&4s6Nou)OP1t2W0Qe<4&Dn5BIizCxTxC_$oNI4+ZhOmmZI^j#cE>$nY|+(0Fc$^7 zxPC5FYCe8T>R0l?Z->h^yEg6aBYGdtH-Kz}p{JZnB{VHeSd;};F`qmgEUqkJ{bfzk z>o5`M3u)%lA>B5;0<8VcAbaVphR|rnObHdGHpN zAf!k_LvB>P6qvj||2#-)VSDxpEZNJG?}pX=6V-hy0;V~`Y0Vs)O6gEIIug~e7G$=A zP7Pkn0EhQLG4zn&J+Fv?Q0X&i%U(~u3q%6odMtW0)0^RFyJ}~t|MO!5m!L3CGOS- z{`S!$%YQu|7{ND=W}wgryur;`Aq}S_ zVkDb<6JaYUFjkBMU3RO%4`-}#~V#)|1 zY5%QVAPm3q21#jO*B~IMQVO3-0xMlj9)sT_O`+9c{tu*9jHY8Mp@1Z8CxCvc$p@CDvSgCpe!) zuBM8w?yIX2#0o#MJ^>Zwn>QP3^+B_us(o`<>|D$dNrna#E}zy9y`}F$LLc#v1){hhp~;!$!@?IUl`p0;iAl ze(Lx=;9pO?c05xAvx|c~E8fi@kR0Fx@P9T&Df4Qn`4R=*HHyGkyy5uT2X_=ULkKV0 zWDixw2E9$gD$ug0oxA%!`B6gpCFhywwsefMzP)9UDZ#kK%sik@7UmZN+jj3(!eDH- zMef+07u`J-L~9PzGA1kUq{~O#b4$O&lHugNQ{G^J`(x?PJqLfHaCibWO)2mIK2}0; z#`MA$-$g0Wi*G+2Dym+IF#Z_59U`WNE=2t?v`oyRgPmNrJ`hT6H!9=D&}tkn{nYdk zGsaaV&SZ|ns6rS1)4|~IX7n3p!|qKHZE2QrOZyq1-?^S#p8!mY$Vgg6pq6N1#+dAEyU3cg(T7^bHqK zn3b9%5i0o!Z@SntjuC}aE4|rzjR(fN;#&W(lC0y+I`x-!5;f!atv*iqW{0N4FUEt( zX(4H=PN%m~V^fC7M`xu2Gn9u!F1iP}q`{#saNF66yMJHE3H1v9(OKfJfK)JTmXjKP z2P}2@#OVFNguBz{F@DusR!a@)M!_qd7ABFx8q*cosDmwbP7Y?qHXkVK!ZmKtnzN>C< z-l#S8g?cd##gI$=ZgZ1%6rJl`>Iz;pMK&pYLEqz0iuf$!_MJ2pvYKc9N0ekek0h^qQVO2pT4>1)7w3q!pZC1t z5yKe<3z4{lI<_kM%6@C2HJQ53T*AMcxThh$DB}HTrk4in4I>P#n1*}$LXvgDn}XaD z3uo{&ik_&_w&DGWc1t--jN0W`1^y(EOP_#5=jX-cJ01E#JGoG3MJuPbV&AQcSu1RB z?c!8Q%UtoEe;ww}-=6ao0iG~4J}McR7Mb?WBM428=fJFvn~lF3@u|2)fcD>PdftA+ z2L_ed^LRVT_ZS5)y@uY*m!lnW(WC%dl%W~h4q&o9BF<}0nxJ>524&=`4Q*l&gUw?# zkkjHcMC!TK4-cTcI)2dkb*mA-x|2S@9SslE;df_`RU@*^$)QkH=SjgKgeDFVchU}T- zlV*+>=8}PZP<*>Ao#?Lf&oA>3IAPoC@l^jCro&J#=($L9m0wc}VUY%Y6yrtYOs>4Y zp5YELv(^T>eZT}Z6U8$}Z4k$>1sTc_e(o4RwZqH)Df`M zjW%bY;hK3}5cIP+A)oF{9os6-cc~Igsm813g7udn`m}ixL3=V(rry252Q6dBP+7Xo z3qdEa7^f@CGzHt$#$vY|zyBoY1E60>95nCKM)hO-67Fxs z=!Zh2#OZsxL6Ib!NpMtev|N z=b^_Mpe=}XDFLU)Y!jBK1=qtvoQMxcLB>%?uu7!0A%i6}J-WApjd9zF70ZC*2$*>{ z0EM30d5rcIU@Bi6UeN1eC;g=&2G2oy<&Po%6Vo9i_$QOAI7Q_{Z+^{kJ>)(6Ru-5V z(ZLaGdgbl+VN);*B|Za;^uqyx?$N za=$nOEniHLCHC&+u5$e1crnjVwfw?sy)A=Qab_4&X{PT|dleq9_WB4Jt|f<&#t-5f zVQX)({&+p%)RcFqo-dNl@WxH8$FVg!;bwK9URli&@e(g}P!PnIHq#7$dL+mP6GJWM zNWa+Yoo^0vCW(Ohr`!w%#SM?i%xIJAs`Eb|dWYh=QkA2BE9%#W-ol&xWcc@OrkJmm z$Uw>pY?u5uUDp-NKs~vw&R(o@wq>5Z zQhJNs(!$73N+wz+>a4rw4o0|F<5ek+-krWbrG2id7@75e?nl^l=LU&!R!n5kdTl$m ztz;V|AHS~f&$voSLSd-dnqk4yW5MYG+QTk6;^!cL>|dD!MOz_6B5ff43{M*SJJ{AK zK&ExzKK>Kgt49S4mhL#Eix3C`NfMO(MDE$Pg;*uZ6gTLhE5gw-)|9^XKpRhyxi7wCEeiWAE&pqgZbvo@#;N#L?F=|I+j#NoY$y5hcAnT{Jqgi38}jDEO3GpqQb%| z9$t*ZA7h69KDBmtFD9(TM%={I346}UNbd>aHUE1@2o?hoWlG)iF+6IaXO`c3<#+dV zCS36v6#Lb5e>UPyqO03EWkgs?Fol!H^&mzq-;m62=KUa(SY0(>zt~Au%BDxfyVipU zLH`Gu{A6T!yAhVB|0$e zk{SaeBHOg@fEg~Wbo(7VhSp*p`_HSx_H10dbn@=?pO2NYd5bz%xiRYWieal@hVr~? zACLH*-I$1HP%P~Xg3s7vM6MFakzAdkU}(e?}5{fAM%~r8eE!O$$46`{3yNg@fQt~vT2;} z?Ll!I*<^39WvcaZAS#X>fA@IL2;8B3fKkW;)FIr#v*EICc6-++{sv3VXewTtJ^~Af z$c9@s|6C91J~YN8jkTR6bYVaYl!gsth$?mI2QbDf_9F?yrjMHs-1xj?g9or-UEd`3 z&8fDgo9)8RynTa>yc3Wkuj}h6wEX>WqsYaP_JgWd0H!1yge0Mv-Ug60@5c!Mdflms zWTt@unBK>~|D5WwM8Vgc*)zKC7lJ~!EyaHSG!+O%B{87u!CgJo1~maouL^AuRebB7 z!oKZcj&*)*ugc5`=`2)ZmBsG^szMP7zkuLN?h{#nLC7~kPUs|;s9PPltsCm0RS_Wm zfNwES)KO~g5x3~jL?3mPMcQ{}$mLffd^=qCk(-ta-ewO5*Zx2=$^lOF5Xn7=$ic!K zq@yGr{7Yu$smRYoKgb0GKKx`1`$*3M4d%ug#xTSn^CF!8UBKG-)rOToiH`~rFuEcn%2?uB^*@XhZVJXA|m8^M6#RPGR)zc_6m;^uKN#SFJ%u*r`jGZj_ZSAn* z562%Z8(^t?==%ptVUwoaz(%}p-@O|=WLW6{+l2h`8RQ>}q`r8Lidy?~gIQ=yuYYiO zcjs5a7h=iSywsYZskn`SsPTS*}9iRQT;bw`O@J`;pu9|51j= zuyM&}`!G2CFC4)roc)*lVFv^bOZjx^h%p}jx3MGt7v1>w+oQC`EXX!oECYe^j~TFOj@)3<`4>J7SYg=njX=Pl z!X1g98wZuM1UaD64j@hxW?7reGIk++`2gOtZ&;eshXKBDepsR3{g@u5`C5Uq$8Z5Z z-EhFr!fK%II8m&zt4-;55cmvVW)FJi$B1KDS}tTFi7DLmoF8CDDIy$dCXNs?3ZirC zB6c-gAf0HrwZkW%k^W%XXgeOhHU6?cp2acw5M_-0x|rBqRz@}>%MU9j_$jQ9@)9~l ziZ}mszj;vhX1LK&F@r@>bk3t|Nl7n#243(t>2K)jMxkIkPmmQvUr(~=kAFIjrUMlJ z>=0F%66j);P_%7_hT|atPzlZh$_M`w?)97{tDr<;V&^YGVbYr%qAwnYliijQF4bTf2Rz8Grp;jVs!g>n>_iFH*{DF@NoSIwGLavU$ z*M9hhOFSJ;M3i*^S+<0rA+kvLzQqKOy;$VMZDOpj>-4g}6LA2}8aF2j61?-UzJX}{ zEI#A^(zU_UW&+LeUbjBdPxSHR46{A7*`fySk%%*vTqMEk!V<4tIgP1icN#Yi$9``V zww4lF<(3E1$pL0v1}`D}@)G%#?a77j>Ly2T?n1`w6|(;}ou`Mb#wXyQx|3*| z?Y0Q(Fjxul5@4p(W&!iut5(v_CtoCVypnni>bRPTtPJ2vSkXt=nI0rB%qv9lr|dEy z`vC|DjaDa~!=I#gEEi#hxbhJQX=c%~T5A2saP28(F^Ux2;E2T;A?oaQ0a=+}F$cs~ zZ9xkla{AYZFZ3oDIx;zv4?jv%W?Zn;DRvGKf^4y za(3yg$^9D+MrmwY3NSp|AF@c6XvW#&Eeeu;7hi614t?rzNsb0;Ls3h+z5_EAWqc1bGkF+p|h=)Sa*8~lqq2N|*sE%h~r!q0@`IKi$UgVyqpm2?bD{EKq zoIMT2_4HauY{^ani9Z^sHCP8d+LD}b15n|IebG|!vP+Zsj{~kqmKO4>IvRz3} z#!xMm=pLUt0!8X)5X%Fc=Vt>e@G_)LI+6*CB%C|2jDRoafo&-avjD>B<_LpV#c6*@ zXnn$V6A6awmJ=66mH+|D;oU$J`A0us-%20^97uM>y89Sz5io)JT>>|G%7+KPL2+M6 z^RpkPxNG7N_Yj*T4Dze#o?)+qK;h-36D3!E^JivuU-R#O{pllY7atO>3Tjz4(+)_H z2nma31CyetiRXc_dMdMcc@crZik)Ma6|ADZ7 zH8kYkMC|Z6%PDX7tIT25O{rzt%=X*3huHbb{7&avjQ#9x;F zEvvgJX1u6Tu0xFHHPoJ?O7!PM5CXE1JXj*^|9tz%`JW`LKtF*BXbbUZK#dI23)rJt z#G;6tHy;3k;abuke%IE|N(valF%8;9+Td>Dfw78PLLL<-F}Na>5W*Khtx%3H`__#a z4=v$fL3!sdcyHIp?63DVGSz`ovaBANC;eGjch~i1a`Y=xc(>ftcw^voluReW5;oqE z>TYx_|5`CIART(p^2Gi1d-sq+hE!^Fw+VqVYyo{iITT=((ohLH86Qa2!U=@2@m=&N?4sp& z|6}O1;~}`1VKcb(`ni~Z>}gvId9t)KC_~}u(uV@-{z}Ko(%5Fshr>P>|M-wE-s6Xm zcF2b8PYT=rL~D`e4KY)TUJfvtgCW~%@Z!Im%eqv(CjHZVWmUo7cv{8Y-}83KF3t5= zq3`ZL=wx zX=$&UWy@Dg_W$?q|E;7bKZt3a53v~fY56=g(rLj1y9}$E-8`Yc4oSfXp-OC0O`!H9 zSEA~!)B)2r3M6mxCH*QADQ`?L^|S~3Js4zl@gi)g+glC^)38~B!$4s&zRp0w4KKZr zK6v?RO4(Qs9pG{6DxC2+vv$*_Jw!?Y0~7)~Z71VM?=PfTY&drs9!rX(Tqo)oB-eYw zi=a?MV-ByMi~o)|&4umZ$zO7Lr8vJ`Z%Urbvcg;At0PR2ug}7mh4^(Nsh!+}#!UfC z9pbP{w%E54Vh5_QbvoM2oEp}_@DWl3*(~F9083~1zbqA3SHLiL1iV!U(fD<|SRf_| z2JBo-o}bYS7VSFlw5D8c=-1cBdW}HJ>Xn>3uu10?*;-?!)r*l-;N|3(nACs3_)l6( z+VR%Ebv1?eFtGss`SXzX*9oSu(fxZkDYEXr0tZE&_1_o+7n$FhL*B!`S{w==-@qIk!pk z67=%~Sp!LQFi){W+^lV|IjJ0VK!rV1!S@L+B5EVvS~mbVWSD`G$xr-wAxs=UPjTJ2 z^c0Sq!N8Xk4EmaH4&@MCnIg(}xE3A+;Gi#KzL`}*v`tte-4s>~0>opr+y5st|ilyGZa;D~?~f3Ro*x$-pp5Cp&zcu2hKTxyn64#*MsN!G*R zYgS)HHzC{6t6Gs^DGE%Y+BiMRFXfJe>T(EnJ;Im{64x1zW4=d>f6ag53 zece!xcfOnWTS4F#*d;=-b0y3jSJsQ@9SHBlYNSI8;mu%i4&cvcfc5*k(bM?CxoP>e zU2Zku990hc|Ayj*pE#Vu(<7FU0eZz0AmRF0TR9} zcT%>FAfm`Qe=!}>2yx#-O_c}d!>D6LBLOdn;upnxDKuuf610uukE>*93a}{$5rOY; z>-8>-=k^erLg-q{!Ku)3uMUzrh`KCL%*5X)mU}z0b+sqt;O(L9(JcW;;9I9xPVeYC z2d&;$1{#%5K24nTEmAlOaVxTG>$!k2k2f%3#Oa zgW0M-xrywj4_$Gup4Ex>Z`}a;iIHl$sX7*@-tZe&sXiJ#5=j?EQ}ENtldsmZkAQJ| z$v8QcKrtjJpEKU-z-j_GDQ1n_%kwXE0gLj_Es1z@6iqajdAQus0}Q3CzH!izZ1T!V zlK~A$rPZ0##30m=I6V{b97q|EDw=Z)R1;ZRk1*8wftOPjYK%j?i9A7u?zl`b*;jXN z_C)6jYX^LbBq=2mznM@b%FhJv%J&k#CXD7rB=K+}IMVXSBq92WCa9 zk-l6&5kMfJq;Ha^ie3|gKd(qAtS2p{CFb1ZQbfemoX`l&&1`^>?aR^cBb}E3ZLW1q zVDRzBH7O@pOs0SR0mtcA^*nZnKU>ii>@PGPbZoGp=4g$uB})u0r)0{JZ6U75f>dUp zfh#4AFiG33MPDwTfu60>XjFY5;0r;vuUBR_WT!w-*z4a7G~>yli4&DlrG>VbjQ_rf znRFdUA6GFc$#jj8c~UU|CW3G5vTGml-6X^Qj0A0kxiFvt!>>F4#e5A2sp7crq6~#0Mr@8b88~+9NOY*-XuH8Zni}rfL=e?p6*_lWV==qi_BKF8-NX>63bzgU1I2ixHUjb7%)K&7;LRiQ8P6J#kU5 zl8*LBNGJm}|AfnjTFX}eQ{cvTL1Y0yN0DJCh)r|evZ0~gsO!Zu=1q7LNPR}Ub`?mb zK6jd`*xPSS!-zd6?Y_i~IH<0EM+{tWkFY>TS}-&b!F*w*{@oee46Z=yl6qro7^#u zh@nZMhe#rgp6w;iw5>;mS(S}?ErHi4#s2A_k=!wrWF0BiMlHY5FIarZY4Y(45$4;^ z05m-@)n!&PgP7^5iG%bPZo1Zwm!WFZgZZpVZ!M4I#E%Q$Em-F`!g5K;ka>?uNIwow zDUo)g++ETY6ttk$QJ&I6n9xHC*)wvR;**V|uK=6QC{zywoKy%Djy}jE7N^u#!Np6@C<^*aGuN5R8@>Wo% z-W=H%K=_izo<@WPHS!j?n5fyrd^R#7*#kLeXF?7nc4 zzkhPsd3zke5LwV@yo8k%2Z?w!cW)Iul!>hA>&coyI!f0fU;U1WMY2<;%Ye+kV8W`5 zn5A>l^bmsgDcYLW7d}mn)=`&kgU!W{FqKEN!rJ9$Z>fN&@j9bk!+UOWa62K&i|MsH z2{c_!>+pihjP3YxPR6t-z!kq(L3aWZj*#)3J7G5pVt%@jWC+5FhPNcay?2dT4LB%F8}c&<*$smZzg z7`w}~6BSikZuQRj2>DJEZkOjoCcC6*1UZ!$62BN6^1P!={BF?F`V##=k$r}Z?4|G- zxt%HmIKX%u@gya0yh<`1u2b(VXqqB`i%fmvBz(Avizo_qjAOFe>JOHw$O5Re$BbY~ zEnxXq{;6QpCT)7&oML6EXl=p-w`}=)``jONIe6Ty^sBfG(T`~PjlTq(I@LC#MVRUd z0L@f-<1p_N#AX3~N8E0eRw$8cI?U16m^$8YMzB&faq`$OG`aXz6(qi*a0r&?G~i0r$yY#RgNMu|bb%6was zQaPfLWXL)#VvxBm)}I3eQQ5Px(Za-?gIxGh)27{uaY{ufkWJrAN=pp$o*7wV_vzkt zrD6l>x}%be4L3L(*1LtqR#jyI$RpLwXDg}|yeGt?&94sfG0X47w77Dv5#^0}$2eK@ zUd16s)X}ioDTf)`1tfA}R6?Zf^=^q?CjM4nVLuJE3_7TVXcAlhtbSZelBeOjCdD-% zccmK2rGgds??pIN0WBvFf5@uucoFWr5`BE&Fz(@0I;5(9O0g?Sp1prck|tGRW5X)i zNQ09meTGdxKymO8R+ik;@Fo_(CyKL!p+|2=5wPm%3e@ZA!|gfEKZeP6F$2}m1J}2l z?9*?WiL&KBw%=~_GBzTB;hJoF%&_* zF4X-64x0P=PT;wk;%j}KL1gX2ecfn5ukMaRM?3XB6Gx%bO18frMRRIc5;CKWL*Hpk zpZX85QrH+tPv}bb$Ds`4#xeqh?Z{YWJKWm%Vbu?8tbdP(s#DR7+N=I6s?!HTf*|1P z2RR?ME&3yBU=%trxxko9MF}@}2#Cz`eGQ{C-iffxwBf$@V~zlpG7@lB($U~a3LYGX z#aWBX?i*2kiPq8s_I;jfbgE9c$XG-B+z^bb0Y(VekHVjR^|4+H=X_Eng==oQkPjvR;i7OrFnZqal%3#gXnb^>pZQ(? zo@-HkSvF^%_eEatp`~<5u9?Cm*gOKbUHIxdMh*!p!>8WI!Q83XhhBiV4q#iuDI#bg z!=NSQbvz-JS%d-li7l@cmr$lX2f5{|MXfi_2Lp7{5E3_x8@*D#i;w#LyAcS5Dy^RI zt4H%7>0ZrfOg0rcqQ!{ex6~Z4k4kHJ1GD>c85eFdgzy zyv=zxzw1JPIR({~66bpxYiVOyja$_hc@=3*YX^%?flH26bSFnS4iA&6h=fH;fke@+ z_{qRGGrb$f-~P2AJAq*QzFJPIFRjVav`+_ogl#0KE}&6sTMWg*mO1Z)s;Xl2kx+zL zFBKI0hQsnEl7lR7(XQ~iy%VwopOd$=rtSKqjtUwyQc!r-3ls6We?$h&V8T0UlCVUA zK_Kk5>rV|H%)wNYr#=P>Ta^XSg>R1A3jXsNYMVAv`{3`VpzNsj7uGu|g66XWM@vit z^@bTBwR`pX6t}3JofFof;HSsx8Rs61t&dXB*_0M)X+?FxtffJJ7F-9dJ#nZGV;=Zd zU7x@rk0)=ieXO~`qlsFD2$Uo+7!hqdL4Ae#PXm`hdB7FvSnAZ=GCCiB%ev+){~`}; z;ApdKmEZ|^>{ge|b+L43!iwSh{+%`ARPo|kpf>soAd+0MOdOcW1VyTi5I8%0(>i8} z(#uNFttTFQHQtX)Ak^(cm1moL)&}()z=W6xikgd$DQfhay$K48o|w99pspG8;D!1{ zl)P3!FZA>bE+kMJJ;~ygP>vXNzuE67zurBOOZgiu(RSEm?u(Wt+>US6yV$l@-XWfL zSu12yWMO!q(|+9PIL@kts#Q7t23lA>Ji$`|X~YX6ov_b`GiEK? zvl~A&C2d>&to|%d(8mvGX{+S;jAw)PE^!EeYV?qR*L*OcM`S{3=kDFRPvj6jsb44Y zmIPm>a{vnQ^V|?zpfbR&K!fADY zt{d7St#w@x;df*(ny7nW4qA$G7v^%S9Zy`1v96x+b0xJ89#rl?h`*p5}jD_^bW;&51*rkXuRdnrShl(p7`wqda=>A%4F|jFsix z-Td_fH+@pZiR)a9k}?z)$_kwGSFAA|5b}@ctH!j%REFp2@-XgdIcDKYFd&H&zL0I6 zx6V-6=r8T<+D?-f$b^nbT4RvqHqK*UohG^kwtYF|y3{@a+K}a>TCQgJVr5?AG_VeF z7fW%&(g*znz?-wm3wh6>WsaS?O2Nbtrf13%>ut5oyK%&XVt8bnKcu@XI>*+#zN^}4 z6qzxEKD)&R4&gTiJo|5TXv<(~w4uD_jk*~k>_;o9><f%iQm1Ar3@<~V1gql+WxzQ91epA^D289%*$N3|N#w$clRTrZ@eUB;h1|1Hd zz#{c&6-9{FA&Gqe#M04w1>0a|crQ)q0?M$EQ;R_UOo54s_SuG47vplbV)3A9Io3EO zfU-f;zKO!fw*-_Pivsq^uUNP9Z3K5X@kQ$ucZO_6lC@srR>F`@<;yy+1uil35AStP z?lMrXJ87{oUHGxsO~sVVr?snsMSY0@9WI5>hI;E93_;A3W!<7qNN%jxm`sbRMB-E0 zZb;2OAtHd@-Ln3A3m~*;unT&*k|h?bqB6!bxNWl~c*wk@?An3yuT~TAoC%a0R{&}@IH&SY{kTf`S))GpS|jyU zf-@0)w!DT6cgd8g#S0Zo8OTmO$ZR*z_qv+0Q0V@2z4Hb=hP#Kpz(8>%TG*i7yN9iY zkk3_GpUSv&z&7uV%Obw{RV7wbcp7OY%B4gmn7)Pp>#bf6AxEI<)eA;&J@kF0yd!8u zCRL9BXtjYHX>t>(hj6m~sw0Y^bex7D8^E2~o_etBDpjiYI;b@R9_(;mq|KK8lgM7m zcW*(ve@k#ff0}&CF|s#t8?6{(P&&2%ya!~9m4(OHzCrKTR!vkHf!|~ML#B*-UpLl% z1^$0^cO*;{^@D6S^p`=0G@R4}s3OZWcH}h*mTN&~+>8!=4IoSCllBq_nWP1owJ9@r zP*{o23dQxIC5Z5riuoO12jwjuTmSf!i4YV!&cY-8L8%&ZJx`6VnCBXXxj=wpeoMXe zgvDdq;~7FV856Wj%8I7_CiA%QdlXyyUjg2Y9mm3(Hhl+-&MO5+ykOt5f9?ldk*;s7 zs1r}hn+hryM=`u&T5#UM6De1R)I&_mBsPU%u=gJ7ir&;?j9mirwHK<3QFmqTHj31< zz`y!zwLo|RsRDq;J&ibbr}s7|i;W4Kiymi~O-rPiCg^xmDi&YomrF=f^^dvKG*z#u zO;Vt+zT8oWZ~BOP$FmN2x>}a!+pLM%wi^#>eRL)E9qoh!Bf0YrO;B(GxVIVEugvSY zBzfh=`8)?}0k|L)XRwl{&T3t+5{x~T*(l09d`Jw}_U2DNJE)p^Zl4rSnuv%TDJ?%& z1lD_^$ZGEpGFa*-yiwvlLPC0XAa=!?9bIEH&wFfBZ@Y?QKb38-OA3kjGFAplFBITK ze*0?9(l@1@$U|LLhl#wU*|OZ%$gAhXk184)-gS47rbZh*eVkhRy%hF(+4!3)|Mc(m za6V-cxKsHyETTp<{a@6voqzrKMP2fKH@k$vDfsEMawVY*mq(pD#2rE9=KYd|L+wKR zW-1malw;FYnE&~RGV>b=o?t#?l28V61R+93p)HIF?E&j0#CZ(1zx!{+6}(!TYqPH5 z#!7u7Wg~+L#keMmCRZQ&DmD{Vl|g6tlz(XWvixdfUju>NiE9#35|C+~GwVyutmK31 z)68EVcX6y8KQ=H}eu4MoU{w=LnFnT`)9gI$u1*cmYBMg7bTIoaThH@6)b10LWiAfhm(|QJJfD0EPMnbix-#l`_uU&m zZo+1wVZ5!yrKH)dNp(E2a8=)U{&WXuJ8&xC@3k8+@E~@I*%MIs_yN>u zp{;uAO<`3IKlcg;hOG{ekHwpw3{>?oMR7T8HfUz_-aN2D+S=2G3s;$nS9u_?xvL+2 zmA;dN6YAtW0oP%P@{bYf?2hcH=_uew?FpwDI)1a}TEt4+avt zd9#bta+Y(p~azoFaGES9{vwuvvMW%6WB7v z0xH_5AMk5mn%eoq6VB+ZYKSDi*-6K-Z%+8wUxuo2I7CnxqNtvL8e+EZ{~n%xwdn}; zJ0_E$qw6HOfh9c_YH4g=s*<(XcTvVU^ z)YDp>N8SHyivQXe7jTdzZRNd5HMEQz*b`jb?1h3XNSbd=-&P9NOpE?8JnDk}I^3qW zf+G-%#>ha4c)c3GF=pM4241DrXuB-g+jBWO?NY+t%Fvo1-&Q8ysyOSXTFaBtrmh22 zsSNmr21pU1dAOeJqso6Ju^=c!jCNq@*GBhgvOIHBGXYgsg>xCnQfaR=t)D?KSZ?^& z$J75(@nC78ls752$^MPmr(viV zEu7cX1(jrDAwf^_>H5~XY-R^-h9L7|z)3rSVu<(o9?xhKDoEtsYaF6}+Uo+4s)}!- zqtR)pX^G-TyhC?6NSn8&mOkCv%G9B_H&2(vLn6O;z`A|*k&wgP+^-Nply_|IM+z{P zn6)}_D`{58H<AgS#_wGEZf&aF8XyEynt38tv(J?cy5wiE)*r?G9c>^L_7$h&JZd zmH8|3-?at}Xk|<0F7cD`Ld_g;8mZJPZhb3Z$l*Z@YBHmxq%)$)IHMwN@`6?^wGOR- z$S~~tjaEcq$k~dH?WzW*d({DuOiRwaZjgvbG_P#x+nlbE@<_CQTng^AH(OQ|M}T-q z?#*!{7gRoW?f%*z-HTYDVbe;-b@UR%35yBwU(a}gy7S%3utaOyw2Ep7-mau;C2gmw zswzG(3}lO|1YJT$D^bU+15?a7y-Tc`lTSMx1_j3*?m0_i2(j&i%yMMi*Qc8$20Ol` zD?Rq_WIBQ5lol{uYoZR2(F}1-EUf`}XwV=6e`)}G?Tu}rek^s>Tuoz>F+bK*`1Hsi z8>*hYE}Tk@r^-ks z*xX_7dQc^=bMe`lfjou1?pA)<$AcK<83uUszUv>pFG@4OSa&|^JNodv^#aS5xi35U zx_9yE_qjLY!4*}yzU{k%+oi(#?rjwDRS+%>xl$!HVvlmnL{kY0-rDQIm|6Q-qj|3G zReZGaM?UCRpRk?6)gnVkYQh1UxhK zetLgy!4u+TS9^?XrSKYj#^4_M-eF(gYwOlP=WW>n~e;q2jeq-QDl2Kecq`TEWW9aSj#=BHZZr*+nyM`6AFl%nWV z4yY_`4wzV_jct5&O5d=VSW`GTJ9$)5DfgeRB92EKbM()iW+*Y<5*hPiDUM; z90y8A9U0A5)4aP=AYlGLX5X!ESB5$7Vl#8o?yTuhgxFnxN{SXX$vEe>jT4tiZ}^f_ zcHgY&{4=)BlugL3CfbFEHSFoJDj1%YN&xM-e$x@NpF6zZ=5F&&N<%d)@Ns`6cZgf> z=s$LCuddvBxAO9hKA@->S}g2W>=wbqm~v>tc-49hYXX^LOgodcjA1ei_tc2ucRF9? z-sJ}ob?jM0&>UYMDaVqbNBo^sF6f5HC%ZGTXC_vn<3GsDozGe7Tt`w@~JU2Q$rgOd4=x{gi`J!Yr$Q$sO=4m#!1>;$%b!E^zV(MOP?vNuNi0ilE z4ULlMfr#zSY}-!qsqE!c8s$B81VG4lnvaanHgMh72h(lmmLM&qigNnZ^wSAPS2nXt z%-*SfpyR_a#4x-80M7H~q+5=Ld0M(Z=o!8_=LPQA;>PUt52`?)F^^J}9LB#8M>o7- zKE35-=rjRF+JqKyWKUe6;g^eFmg4oIV1(;UJF31cK@C{%#NqfQg2R)u>ecbN4psA+63xovf;+v{dwlJcV_mZurqt8 z?_P2_^}aaeZy|r;7#C?5Gk!Lqjre_!ah6D&2o>7%^-=J>6~@Ev?vbTC?UwtnbGdV! zbht%(mFAsP4x=Z8d>jw_y4X@^pNYIPh-YX{S#o0T4((vLeJ~`CTSYM?lC8w@jz)W{ z5^JSc!ZT*qSND4GNhWvDf82j(GM;&2g6>b7uD1~$V;-Gb`Q0C0Npy)UdAf>^x8~y$ zH%F84g}-n9^ZK~KzQH!M@a;v9ow+VB=tE^@m({Ydji>(Jz;*ees^83=L%QSt{2x`{;ZODd$D4I*4zkxV zqL7d{WE@*1Dik7_DeE9Jdmh~o;ISa|AgD$5dN^RZKPzo`+(*Lagu`#P3Y{d1D;l? zf%!B^tg>+_{~|djTB`Tb1-&~Y5&qlgcTKjlO!#g56kxNPLvzgG>8Ua=dAXE-qNZDZ zayvEE6px8es-{haKv9{SDQCOu02?~pq>l*;0*uV34Y|)kQExSltSMv+X3$EqqH~{= zlp>pa69f=k>OdtZ2A_4BzlC73SGAe`T@|*LZ)LzfH(FmuAf`i~!K?c;@@fa$c2la8 ziq>#zKfx;KZH*sp*t3XvuFH1tU5h{zG~^zZge^;;Wr)uYm%`$GG-+Z#|M&jl^v0P2 z09nvna%_!C=ctn~DI6a#K>j*VPGaSBcHEM-6-QmQ;~p_IZ2JBpN~n%$1K8w;@%BsI ztqK#b1r*&f^=$SWWI=DQ6<9$>3%M|Id@k)+1XfY=W;j=TArg{MjqO$CFIDRDp$zf7 z;-sFogt#^Epw!ejY_wJe3;IY|H&_lx0u7*-@?e{Z(g=4BS)@t97YYOV!(~<`&A=+h z0(bby@kD!{ufkg+2kS<>1Fmu9WlFZO!9xW13~r*w&~{(v0|VZ?4xNoTvZoQ^6VD>v$##Qeejtz# ziYZxV1s^N}mrRpisd4dR?tQiOnxE3=qmY{UbVqY(;YOV_Ws}0`95|&%w$qDWeh$XQ zw@jZ`x|+T}kCGEd-Mup_Mu`$f{Sg)u?uooyQSHE)lb0a~k82pG@x~b)ooRJ)-dyA; zXg01J0D6H3=LPtb?)eS)wanZ%&;P+j;l{uzK(OovgBoV%t+39CM|72h-H}R7b zNMxrrg_IZ4B@AqJ(ug=OTD)JuHr>yaZ+*`t#IPHFP57r`z+A4H0rMLaU_^pI#X9PD zeDNT_P5?+AEflG3dgC%&E;K>{BqnU1RM7}3hSDWvqimJSkRaeY(gs992`PeqOC9u4 z?BE(eEdXwQ;TXtV{s;mP&X2}6nGJvjX9wP0#0Kn=Lk(@PExhKDcS3EE+J*>Qtk9?V z5Dfk1>dEbph_ws&bU#cYB+T^ZBiPR`ZZe~RP6A8e15z2)JWz-rFHc)EzW6}&oC&xN zT(%-c|G?5v3L*Rx+c#_pi6{4HNi`jA9qYd30ULKV=6XejXQ%c^dB8?FIok>XgT>}k z)rWu@7sHGcley=z`U{iUR~G#i8-oS-4AHkiMf^ zFn=#?-&pymL zzd#qk`I5HSqE`H{ZN2h{|yqyO#r2+59ViG zB|@^=m5s&krwPPu7^sxpNZL%#U@Ax#lS!<1BvqX^UI-m4I=Gk~k_s|AAHFSEC?1dL z9@?G3`ovax5Q zTL2tw&5yC5nKpm-=_QH-vc0wIO(P`wm1UF;z8)}~@}?cm&oOujNS$v^X)eq_H4|eV zSlx}S@`MDyk7SgJ4-mhKuy%Rt*Ypg=qdPBnQJm5~*YqyV0klEbz%mk-{@w!!yldK2 zER>Fq(VIoziSflwMoeeq&-axhW*j5@65Y}ex9ZryT7UY6Gi*C<9X&Ek+*LTayt>t2 z6yndixfEub^KWwnbJh|$8t$PVvlQ62TE0JoPrPcGr3oYdG75z_!XYR1zG2URw^K9~dt^*&E6}0~{zhzW;84}{CcQV+y_EqtY z;{9yec6Y_3Aa~o>$R|@LUP^oj^Ahah=*nR?U&H!MjnVj&YZYGY$>Hw`SZZoCU2ig-=(PQlW@ZDRjYZgigA@KTb{%^>8gVAoL;*_Q&^9<`s_}r;TBQ>BKKI2C#Km4)P z3;2fw28)s8kl4iwmi9kdwd5h+P$Byx-BSf2BW})@EG+Vo=O2j1W{@U_{l>(L7_Zmk zlrAn53!wfPb^Q+bb9~Sr(W`So2{Z9NV^lnNjY!c6Eg0^9?P(lEG>MX}1~0XVL~=Td zG{eHj(;=%=30$j*NYpMeA+*8@Ak(i7!*8NWNwKvA(g6tk(Nqg&MmpDYyOt9 z-JoFnxtQ+<3JlaTZ|QQfn>sYtzh%^>z~B{<#BsRnHfFJfyYX~x*d7uR?hSH#7O9D3 z9pf|geBFbX+jM#=1e4!Hn#=ukV&0_Iqv}6hY=UU<`DI=f8PNQ1nzcrF75IPUAU|XI zwYk+Kih4T}`I~sQDm!k)oj&Lf?QMMgP6g@M&o^2pvMADYrgwUfv(-6PDJAlzn`ycr z#kN;0e=c59yjSUb^c;;4{`CK57Iy6Z9KB51d`7F(X>9L&oKl6%3#%9`l4|y*u8-7P zmtNOV25oor(`0t;-cY~z)!Q`7a4%pxR_?M^!H2Mo}H)ZHw+httBeM!;%@gS;(m-7WDz|w56V$9@hYojJ+m)v6P9jXG`O3Ov$XUiP7BO z=yU1G;a*R?_jA8evnaHG-F*4E<0-X!y$yM$sRnGnp2sn4@Rh?$jw8Nu+5+abS?vZ8 zcz7{&m;rr04)KQVcF#VOe}aX~X+;KdmVWO@KC7kp(_>&WI#KE&?m@rz$@rO${XhV@ zrs9B>op?bI^e*m+^l$@^P|?g=(TmDn%G}curHW^+P)u{#&4jO zarg<`Py4MiUcjW7LS$M^lDF5d&%fT2@-tWWVF4)u^EbL7iDWGCrF!EI(c@V%+m)gp znwBSloF_%t&4bb?ap5HNJe`4z(t7CLn1^#r0%o?=ZTJDE05*PuS;qvL=uF`PGzHS@ z8whfLZIJ@lc7EFa1)92j71siEuqgnX@l%&!T#R>sO)DF$hpG1kRBPwQ8XG07YAZ=u z{@(z{`35*zKP=7PKu8E#q`f9U{o9{~G1L^odS4Ga+e55RVg$70qy_2ZOLK{ndUEVS zu#GV!+j-4NkU4uwwu~)K#jS522+@?K*_{d;^)%uaTIkG=CLa!SU^=HSX>L$k#|CzG`md z1?01gq%b2?0@V}KF)Iq&P1q$mz=dG5)XcTX>K2do`b6txd@Sv0Mwqb-iG4a&+HSZ6 zTdRtCK6yb^_p?YX#!_AMaHm5rHdLqqv)loAr);~=_y^BBnUPDj+;V;q&bbI*$pIaI zg;+}lV3Kk8uzO!J=BcWmmUt-o{0Zd$6h0<6Y0<)4pidz#Bh4yq=#sI)ApS}zpe=)! zFLR(g8i$gKi0E@;NG;?xjx$qVMZqA zd5x!o9Y4B?6vORiyx0MoC*r%p$iz)BXiff+>jU~@M_`;U2MWSeqL)C|X8s6NK0{_z z@Cm;L$;~_{U)I+O#3vZ}9$=|jni(Fu>$ll~i4=da5p3!G@Gta3fKPvwyY)_&tx6&Y zR|;>^FoQej6>Ku1OiN&ex%XEk&~g6{nc6T+SMd(O_#^dBSf+F79g#LMu`?;LFok#T zVrDw_RsI!<+y^!Q)Vr0D>7m=?M#@0jLhKV4V_^^KY?BovOeV(@;Heam%r@H)fAu(? z0JOFlD04Ld4!*z$iVaAzF^S?Vd`;3{#p{`z_oIJ(eBIvzRxZy@Il+iGjpLC6ZrJz> z)malXFuv8r>+N3`f=Jl6e1tmKN`Cwa$)pE)`|{&27vTI&imW0xq=Aj#Fhb=MP<7`w zWuESDr;@59+I@XIHJ+?-Q8cS^bC5!r_}XEQV17$-P2TwBz|5~~GNAFX6&#C$uH2a( z%Dil2FCLN-8|9`!q>ypV(s%a^sOFaojLz+ot#((re4qRQ_^bT%s(5=`t1n4)Mqn_L z-}>6!1nh6MFe_ZIvcZE+-)886HIdC;_}0n1w+NezMq3VoPzKy5cOtfkz82ImZyDI> zMQuZukIo+FEBL#Vb`r`4Ql=IuSL?lhX=-=u=tfwmzU*oL#);kU`(gz-Nd)5DU8#>m zHpa`|RLL7-C^#v67jrPM5e4QQxd{KiwlL%Zj+2P;UU`EtiL1rOH51j%K%iZ2l}2%0 z<5@PGL-+dmP6~T~2VxMYHcJo^dxE#|2T6!#r*5`&Qu;nAX`ln*DHp1{;A_DvObVO` zml~YB3eO`#>fZi#c@ZRyxQ(rW%F}j(;$8l?ZIZuET78 zY}V)|c<;FFl_-r*-lrPmJE@|;P#I?|?pd&_tDr+2&6th;%=6`Wib`r=Bij6JEXqi!HTz;!I9@8h#Zdp4H?J z!ID}F)Y+s3Sh$iM(pT_%rywKGXc+G=3&B>pk-GY%Xzuc)k}d=ZfVJa(y1)uUH{XAE zcnT%X2#cO+AS;yu=M2pH}-In2l8V8S>j3(SbJtC2P?jRlDbcjC9mG zemZfkiRvYPTJ_E7OA3yiTrBTMaXo=ITTKH#0-QcD=WuPWP zRL6c=ILEbrwX>WPU~=rQ&L_r?o>`6HnQIO=RTWa-AoQ3Yo4T$N#YW`w3JN$hU{b-^7EVU9Ts>lx=EtaKxo#{ z$|YU8gz+!$rqPYXIyFaSH5`fLrE8IzM3KTZIrmbIZPfr(u*&UW6AU@cfB~|zl;v%G z1k#G{!b1;_Wy*Z{@#v2MKKZJjKb9}TD6rL6AZ$b1@MXLW5UxS{2Z#lfrbs1N#k;rp z!3K1dk}F;hPt3La3vf~C3L=M_yE~oobesor%4391`T=FBL2;5@R#|?1>1T{~LRphX z7QiFF8|qqA@J zD`r+;4P)9DX>omKf;)4D{#}MnoaoKiGDt}t-d6KBggFvNMdbuHt!Q}eh2JF!A<25h zI#&v!b+J>zWxdv2Z?Vmg%+ZEa$`z)=QTK4x`j}_`5Gt|C^hr3p`|54MG6GVuj0QgH z)U(T@_@F4Uq%Urp$y5LGd)KT>E1NaQN)_7Nb{U^LQm-`JPUMSK(koYv2mRJhPp%F8 z$SNOrdiGC#tQp{9LAy*c#FsfqLrm69+x=fu!Tfmbi-x5h2~(hp1Z&Oq$*PQfL9Pop zH@b5wM}xQ07uV$lQY8PFyvWy(Pw>$Y@l~(9dBw~~GwRzf-bbjX1S#byvnU*^wXnu` z{J@v3Gk54cY<6pqhxKF+Ezyp1vE=(_5#=OOP5A$QF~l&}PdRz}%dTdZnB%bOGaZk= zEZMSgx;58id>~`oLDBiMUZqF@5YTFL3wT_KolGa0=n+Fnq_)b#ojc>_3CbWaV(V1v=wCi5lV;T6DUyIP z-ju&F&mH^AR#>DGgAA8({>z9Uchq1qr4WkcZ}vbaZ8ik^9$~_sv;;h=)sWrl0joE;s)s5t1sVH%eRZRKXOulV4Mewr?zXu0l+?V_=M9 z@)m<$FsO=08WuosMj5L~N4srR0sK6W`k}}Xom`IG5{JLlxVEWw|2C>^81awM zX(>+jP9Ep0^^s+{Ooh9FWzFC{KF=%NR??zo=@FnLY>)|P$69aij7&(I!m1s%U((GiKp4y`vjE^d{ zjOksl8z-$BR9b4%{xYfo@bT+sBbY3osSvp!o{EPZxuD?;o} z_FKPayT0phF%)uX!_P8@rHlBcs7%RW#(KQW-y)0GpFa953^lR>$`JWrUMIBE4A>$S z8r+wV-||o7Umf?^(wl0=AvRx(aymVomq)xg^ca~3)kN~IbWGncrZW6gF1dy}X9~09 zN|0?hOC;o-y<(xOm=g1x@ey2*m-$+ei4&0o|24StO2G}?T?o}B~WdTk0TsypOj zGwWWEWjpbnVN6_ZOx8aXk(*w)rst&(&v1hNzmq95EcC@G{CJNyIn%A;h?lwpUxO=* zXK$saO2%?=KQ9KhX_udj`8ztQ`mO_F7Q&JDpy15&{!+an%F$G9!aak?4noy?`PCR9}E`BS8&DHzyRCNrYoR85ch-}Y# z%M9a9DHNI9&ivoMeI#_vkp3zFCadQ{73gz8n8&pFt_P3&V}K?Ivti4>(_Y1T+$Q6a zzIt~2L*K=b&hPa<%pXgbN_;B!CQmdE%1dlVNCG3wL8Sk4p7j6&MQGQZ8x(`}wi`CK zey?XVZ}&Rv1EAw^{UvWO7;)+Z#;7Uz?QwQ00r&l5rBJ?B^kK*v1XTage0J;=y!bhp z&6xmeYJ%M{&%lU}DoUXvf0A|mlHX7eT+yyVk;3840s9K{c>{DPFT=VU1ip-L+2~Af z6((_aY~w@F=~;?NJkxBGd5w8vjg*Z%mkWtI9g$D{RfLncrR^8C^25BWuHF1@n&!uo zgsbjl5y*?*sOnw zZNwnZ4X3IHOE5J3jokLsXTxL{u=$+iNj5GlvDYC-nJ6X zuu$ZV!p$TGG{ZzcCe(3&?yDfjk5{VOYd7tz|qPY6B!-ZuF$ruf2t!@JZb-+o@$nY<`v2E z9w2kmv%@-E;_04^ut}0$$hGqr(r)G z7rYE=&y{Ld{;VU!OBAx^XBV(qi6L&+$foA7ZG2UxMJYEvww1lYP`&1egV zkc>4l(CqhZOfXpFDv~P9VJModeQ#jO70?ikNLv&2o0=_dDwDpES-hf(Vi#d4##h7HiYbDJ~_jEJN8QdA$ zs1Je3TX8SF+S%Luq#&aaDa1`M`1{j5tBUJ_G7mW$gCY362hoV@UrN%an?y@)xYY~M zA^J~`UVS~j9d=GCNUdt~CQVl)a+JdJb;`98N#{;Q=jZ?@3_e+rT{9b_rN2f=W*fC` zU)!KD*XEp8a|({pLfZe1(WANMESWegW_}1!eebSIMAmg~&QHo)NB@`?p%|(jH_U0< z9kVK(PdHMN!P-KL(Bt5XuGZJxDN^JdK>B(Ri}|B>A3-v<2dpIQqvKu^szP6+$!YMqffh#2OxlazQ+?&nx^2eOSu}4jtLMB#3m?$8v|GgffXHg18nr|yYIvK zY?ofH%lq72JMHCD%{_fo3)QN9v`vJ^R#>1Vg9Zn;P25dqAI2$!EEC(Q1E)vt;iE$3 z`v9`9gONKM8{~1Ddh5yE7i5&p$)SNa&e|}Iu{T_(qorD1ZLXnDeOSBmY{&62A@k&Q z1@q`_YRPu-HaU|TKhL?VuT9Pk6oEx;3NJxJO7-to~tck`$in~_#F9Nwsx z!eu<1vlf)h?#VBTPu-osZg6CkxpwDNYSGgJu=my~@v93>CerlMYZspyNPy@X^8GL# zyfc2!75sNZ{-cst4e;0TUE_7-Y%VGf7exZdyH?-u)yx`#5dZu@^YYhMmmim*`szXd z_|4^RE8$)}aiJf9sm?dFoA(!jY~O z?eoi@nBs3|`mS%X(32Oai@6jB9RvBui9_8LPTb;GLTGaMJnheqJcU%VZ?-tRvi)FTp7vdKHHoQ0mFgwY z;(503?PX)8wEzBa!pzWSEx35!)57Vk>4e0_;~Po7Jaj=Pm~Dz?F=+#Yo0)bcb1$>7 zr_P40g?HO-Ui0dPF3jj1`z$4XDaKY9KA(R@a$zovWOIHLjG_!!Hag9a?; zLmL0aWn!L*Rb5tZAx$~25BE!KfQ_oQjvu*ZAx9^Y&1M$Y(keZmAzkElWKJ|U8eu6f zEXcz&^_Z6DkVO3=`8yR=5EGvkH@|%^YY*O}a?uJog=9g6KGB#t{<*LmNU;W*b|)I@ zs*>O@*-r;jdcu}_yZ_R2p8Q<@E`6Rp`}`(LS-3+X17%TFPY=$!+_ADD%4ytq=a!4(qO5xmJpK7;MuD(VhITv=w|tGjl9v>E3PfsOJ;<*a z$xVzbyrK?EydB;*Du^18s7&S0ZWDPefA<|#t0zYvj*@W zw#ok;TX^cnMI1HQ1<=4O1t3kzKfu@{P&NDN_XEUEPp#|?EYm0sAwA>$Own>bKa;y# zZUjiDo02Lg&EMkv_5lkd{@%E>>DoP@eOd>{hpk+Q3z1KwlP0i{i+eV*m_LBTY`c5N zZ{sH72ULtIEL0Ktyfph*DLc9L6JOf~T}@s6_J-8hY^l8w26{9 zwWj4Cw2MZ8V)ngzla(JJ>1E7TS0;Kz;7v$*j?nP~2yD9Yxs32kY5vIfUD17wU?slx z+9Wc*Zb(MJUZ+l&Ti=S2ps)_@DbjIt4xuz?pNnPNzp7F*0|4ieX?K7bMS+@FRi@me z24L8~PTSZHKmKII=NqWXteiJ>C3!I64h`-rgp8D!h&~}a>-P59Rir1$y(s^CZoeKl z`m`%W4Jp~RG3d<-Nq<<6;`%(f)U@`Fcfgq7TV1g69(g#92RXYA7j(%L+E)g6+9D*7 zG`*o+mZL>c0rDmhZ+=btM2r`qc zKVQp>@$5)_s`XWB(!!5AJ4sC?I~K1 zNS+6x_xQ{HNx6|mpVABb>SSap$(UeqN=oOQ3dNf_!C>7LR-W1+*;fPG0IX_nt@4Qk zu48M}4=j@rE)@A~@SPkLNl?f2?1(0gPXyM~2_-o%$gIYN0J9|ACA*)<-cbEk94$6#!B`l4U z1-TqP<9=b2^1a*fbjaZy1TI&z-nzZxqg4CN zVn6y-Vk*G9o)WKRQhE|g^W*W=b zlt^ZwdhU5HIS(qh6*hRh)?fMN#HKWiW}Z(!jvL$E0H|%|@t0PjKA=HT|9B(O^JI^+ z8ius&vcR`J{AlANRWa4cVmm}PieE5H&fAFd4~|*papnB%ZI%&h!JOYOf^_NL<(b` z_6NqE0T&$y8$M3_A9WFH(o_<&`V+d0C_PMXC!cU7=ILK=J*IT8fy^U3a4bFk_h|jSqZVW5$OeI-P^{he=e=IubOD`mO46_EtU&tadq?jwtRfg~TTZM(S z3!cUr%r7_p-ti%&qWB5e2I%7{!b`ndjEvfaSzFHoIN;z9zU06Fe)J4TSR}saq_Mu; zEJln? zdkA=7kOKZ5Jc7!Hz>xN4m?|~Nx&O`O!cn|e8-5R@}1!#EV3*@x7&|$jC(E6Ahjd5EB3kb&uCK_y=Wd1Z2vi+DRMqd z_2HprXQ)WG8-JkDP!({mw3pq^(ALS`PJG+E#mK_08E>9^0AXt*+0@rwa-;-#ZwR$f zxnb&E` z7VI?J7_@KIGqBl@hM(iFHYKV|Yo_8T5ZJ;A3UC`^GGH?d3_2*oZk0F!D98x zV_kQ}^r)K0WaM=tDwFp&Rcym`lJw%eiqQPo5MZefD^8iAFqasb=4+@p7{!vh896zfB}5&}T&mW=uDtfTn?GLiTX=h9w%EV1|D z(;AVt>o$xW{>paTnh=1D=|qWJG$N$cM0l#(2mIfHe?Ui(Pm8TNc?B$QR2c;Nsw|G? z6psbUOphPmYavt;Sue=?{-!2b;|az{iL8%|*HDzk&LeT%S7IWN-PU!T=FSJTRi!#K zV~#}qMyAZ@QTa20+P?SS#U2c-~~|xNSz2GGKf2hA0l5@#k9*1|#Eq4%sH(zVB{#VQ#)N{DTTHGa7li z*6JN$I6|Ve*Y1(B{4-Wc+|h^7GS@8oe$pX4i0m@gMcjt}@&Bqj&u!3d7%SjCu|=7WF&e(0$=QC`~i-~dywb*i4CpAs?jpyCZ@aD>1IwZzv#FEX4_F@Ch%e@UdSbPYQyeGf?silWOl8bIUrCA*2`s=S12^2~ zC+&u{Y&qFsFx#YEhlNtdJrwzyclLslM=jfDW-_aV2*dk-?@WA?R%=_?{{X8N5NGBS1 zv>2vZ&#;PP?u{F;g+V-V$g&AU3HU>W(to?$Dfo1P)pMQrZNBN9*TK|>`*RZbY8bZP zW-Nh3IP)pF=8zlYx}i%bgc`qJ$1;7*VN7*IyZO^Um7JfQH}E%j1K#Ezek1$tY}L&h zQ^@KP%$tFZiBjDtj?8}CAI;9Auf>o$FYFX$%GV)ddatsPsA{uXMuHwv9MbJ+pq2|J zrWCe+r*aF`607HDn&`rCFwxYH)J&s7uu*GX@TXBAzKe0$fT$W?69Gu&_cM)Q!%ClS{Ecc%MTtf>q(emk! z59N7fEk%k=f10cpk#6Vgbl3->nW8|<5mKtEo%LjOL6O?sZ-~sVKA7<;Lzz&0)@!!%SVjL8TqqBqij0T!hZ1K!l3-M#w?9(Opcto<>YDyi!w1TiJ1NmiJLu7lI zB;u|E$7mdX;n16Zzg$J9yRM3m08yFk{v2%T0lewCdQ{vatJwiEB+=MG=)dt4YXi9K zWc)j?rU}_BXWVgWnpC7LG)5Lofvu3SnRP+g#^=QKtczHqr&i}3Iaw(0{Cpx%sxGe> z7mvXZZfU0~NMgMg9xNu?6E!b=+uMTt(B{%4`f>33-j^KZ;l$ifLtaC2vW&P})At;o zsG^X*ki{EMLp>drVP_@N8|sk$=FOwUXO%ytYR#g!4csU-ql@{KDTYe}Y(c@&b_;tS zNd|#o43lo(+)NfPIBOsAyxA2lE9NdEn=7cb7|qY+-fsT|cwO1=F`x-OQtIR$C2b(z zl=3QsZ5Ls~%;b`|&@+3KdNzF!&~b+jm&YOpnmN`)&eWzE*-~-8lrJubhnj^ZGDiCr z1d+_fux}KNRs^5ZY<2Tq~vOLP~55u)>mBNm*-u3$7&@?~9 zmUcB}CSQ=VOVl>@%a2#ovb`f|o%emFf6(@%oAiWpM4HjOzM8xi|115H=G{1MnnGmz z-FM+WOB$9sKF+zf?40vxZELwrT;|YpI1@*dgop*rlUYA$vEk12j%^4? z2URD~lE=X(BzEt#+vU||$=X-X8#LBi7W`S)alSVv*;6Bj%F;zwOxW{_EEXx9OgMi3 z!p;usoD$;D)sr)O=87`YOdnu7U#m$by$8A=s`bIxIj#@ZG_?aEr!Xu1c5TYK<1@Og zREfZn55zkgbk7BaInhgkX-Y=SGvysaxph*I$+q8V0b>2meYe#jcXXniCk=razPh3+ z|JVZ25zKdSN>=%`ituR5-4{u9;e3lDYnokxJvP~)dc}v8I;duD@ZwW_`~>XF(H~j5bxo-J zSaZRc6@%&+&m9ncm^y4|r11MbwYUafVj$W@YxOJx^SDWW4!wuM)t+|d4ipWn^<>QD zJhi=oZZJ1Lco}qceLD1vMSGx2KO;hjazQt$TJY)=7Q`H$DsB)Ggqz`8KFE>!*2rid zY-cc(N9DM#^d{VlDM-clseO;8qMKo=O8QXib2@w{j`cg@8ZTe7VWoc4>lfX-4FwykXiq*M8jp3i zXKud7pXmveIWn7CQy&;czeK0a}oTB>JW&hAmf(24maElrD$;T_!x!fvx z_QM(jhEXK)kRqnUr_Um5ZIq^`U1LDFh#y;d_O?WK-L`KeKRBl8zRouN852DqpY<#_r}h7RZv^sv_=g`p7=*N*73>f6sqI4RhG`~dvQU$V`WgvMJVP<98Lx$DqcbZ%eDTtFRY4BDVkI{|AeYGfXrgM~L_jd* zpOWxP2#CN5x`ow3%j|}?ajVT_iZvX6N_sU1{<^#VxDU!WkKQhuEQt_E%AA?I(dK?h z9f||$-8ztjybR)RaDo#s&d0fUdQ>il1TTnI?2L(2laywYRg1O}#i*DYM&E>ZI%(Gx z^&!A|*iZW*60kp3_W#}*%iAN><#s2O_$)09yzj^(2CYCRZeI_XT$C9`biV0-bjwJ! z3Q`e%yuG|<@8~J<9I*&{GyCo4e3tKM3IPh)S*QJv>mU@Rik~BSmy>zQH-rBJRmeyw>hV~EF zN&6Yl2|ZXok|Kz;>SO07-X40BH&Ojg0k}SH zl|=?h)A%%FIbL0woTse)u+Zr!{>JqK0zVH+V(rO~I`99;2|jRCKSggpmT&0`z`=W4 z_RPs`KG4zojv!Vg-|4$bvk$!ZUcK_3j>$>;g$^USUH=<}a4d(}CEY|P_M#(9a_?@o(R{Mrgc2h^Xeusr1Fhs?CU;&Hm^@tg*pj`j^|gW(l!c zXOpR2y15ng8d0avZwZCD52$lEQ_Bb1PnaC}{ddCokUGe%l`#SF&$inwc^}?nT0O{S zD}Z*>1lr4_{QN2hSbe-EZOJ_DgwQ`?VvGjiD z+*BVV{q#dn-N+*wsaxRR(zYvtDN+K&*g^r){z~J1wXVYoXFR9%l^jVanWu&m*d-rX zfJy9gvQWe>d_(P3Cd(A3${}8K9-?etgZf4;8;GJZ2p=p4^Y1fAM3tBgE0hs{0L&Mp zkh6FgbsdS3um-{dm2eZUOf+dSPQlC3G-?*FrJ2 zuZo)N@t4fInLUc$TxKPWMO=GlH|%KF-rD?G-Tz$xTYRlqh-?PTxF7kZ(y`J{-{v5KNSx6gNwZ2{4)`vu^1u&lY0E*D(SvV)iWF3OvNj)EXNrdWn1! z(u&@$w35G~b(C6+T>oY$aTD^Gc~j}q3pJi8$v=mWGXIPDSaUfTH$?`y2*D>YQ+|ev zq-J>*aY~IJoglnOAfnwy&^bOrVxL%oY!iNv{&|{z9158xIGU%!%fQ#J@u>MztOM4E zl&1gwOH82`SmmzyQ4o$wTXc0hjG(@Px#IIJarVe5i2n22;5+$k;d(eFttAi#$mSm2 zLH-dhr?0r#ANAK1F3?FO_geq}tEImKvgsGLN8G&ShrSAXz#F+gs&eI4bcTD1Mra%H znAFqtCz47iZHqAdbK+ijYSUv}?%UG6PE58l9Y<#Q7X0F+$nTnE`u04(wk%$^7!c@d z7QOb*r@T*bn&kbbG?AVHgtZ@HFQ?uYL$1N<7JSBFV!HVLi&5I~lko^JU7w2CH3%8r+fCziKScF~;wDJ$S z0^w$DtVc9FCdI?a3SJdFg!Js=J0C=Ceh?$wq`!!%$s~tEB%QVz zYM1QC@$>gf(!iu7#|?E>9ic;g*rOlK8DeBJ+Z<=P<&-86K~SiKd!O*Zss5z3+;KRN z@1)s^TpKV>TqZ3O3qn^u@=%jfCL3qc9=suY@_U;JwBgg$-K2O+{Oj?piDj2HDBi%o z1J=yLPEV9{5`QeelTg%KHF)D**E>+7H`dR9t3JUjnq*nhPdg6^`G~)2=e9$~cEK`X zuL8lrruo?%r*161XG(km5_GR(ls~uO1kfbaV0z;alRkJ%)~%}Cfd8`zZigRjT2YjX zk}hns2F_797%*_X!X#j1?!PzxCWZoIxzZ1nuJ6A;C}Rw335O(C++4I1!&;Q=Q&a9wWeF5)<~J-Nm|K6HC_da*Mn^SLM4vO7o{dhu2E`& zhQ~blHvCpBBGtz>Rk4>7so*VA38|DV?ETHpk)B0hlvx08bB;&n3DehUG`{JXVQBxQ0@;v$UgtBTCZe%OmLgsjWDYP3l&Q9^^Jy`&E!eL zrB_Ry7H@T*U0!kCJM5Bw*B;df*CO-ewtB#Nq3mTl@7m~Sm=bwX)vsO(I>M4l$poUa zGi~NTgIL1M^0*x5ax30^ZMjVfGi*vsJ`7u9dQH~Tmj_gYMjOI&Y3b=5lA>*>5k{7E zC=1L{Z**Y}DZy+9Z?6b7D!@gDH~Mc~klBY$s#5Sr)B#Dy(ZhlyC1Oq8`*v|c4r&A@ zmp=Y{VW-+-AT26oRcj(W_YN1$8>yI3@J&7!GZ8p^)a&DD5TNXw1x~D5_pNu$Cuwu+)bIUO21AP0MeQD_^%ANTqrDeTs~Y zX6)#j+ONdFq@)yur!OI-T=!h9cr>%1^n%d!Gr|tdwmF!dHH8cO3mWo~xBL710Z z3Ej}#y2<3Qd!skFQ%A64ch;-nj?5k$ z{%XhCdopEWul3fOB3&qcdJxzRBN76-Vbe~CJVZlA_+8R^kPUq<{DWXvi{E%-kfW$- zN=Bl7%-q3BmA8LaT1oLeq!5&uFO%-L^8d7tlX}UwxkYptYA|lY?@Oivs@qAqVJsB6 zI{Hg(+G;Nny8!xMm0P+|1cTIDl9FXxwZUEt5Cm=extBf-;h(6hGcW&frAkGBNnX}+ zqn%noaa?mv{qHcmk#SSb^3bf+qJzbt2QKcYI9O7kje9eX{_$`Qq@8qJQ8g!33?Q3( z>G&S~a8LsiPGg`n>%q8Ltbq-m`o=Va+xA6J)?5eAoT0TiYf0&CbYxZ&kuNnKOGzz( z`=I|=I!pe?JGLe7XNF!m&- zX}V_h_@5e)JF6xGqFAoa%4##?Vt2!Nxw~)z8krH_&%{(S)H6v>a-Osh^WMR8#q$7NhC8gzWYc=dA6Yh+F0=RoRy=4xfjFi4pB$ z=-&H0<@5mITf(09aG}%lLrhO7ngM7G^V}1H+C?CtP%e!KxIZ;V1nVq>rZtJ=+c8a7>vCcLQN+x-$YKb+auMteoaB-#I?|L6rT8?S0_RFq126wCT|yP#--q-ss`iPv823! zCgsZoRSv#ZB%A$lBAL`lw{aMNhu`}GG)#XrI4xGGDN6I zWoQs0lpzgB8;K}WrOYWqMHHoc>(TjL*Z0pkf1Fc$@ArM4dsz3n*Q!2Rl!1fIaPV$$ zOK1pMjg7d=R?Q1(5}dcDZKnLFLG~x;-Ia4gWTiyYCE#X<0V+48`1@z&7Wv&{x**l` z)B^gR3bCC>mfpW|StIY;y@I@%cFLQ^nMV(vrym>dD-ur-`*`|l=jQ!7`U=H@$KpOb z$*BJ&8>Ub6K>AnyeLEPV-^J^?UNJ|Ma^?ETqnC`r^q1Ke2Yl#1JuGCK1wfKM_DS3+ zX8dlxo{m$Prhqi0kf2-|Ip9KTBM?Y#G<(^pUk(E=`&MlfuzPB0z~)mh4feJx<8bKmMGrtW8soiCVI;XR?Fe2+FJDGir*4Et73t zeUw^naiOc)!NYAO-)u;K^knz<0B>y~QNMak6>H4;jrf zJJuh;a&L{T>f0jc3O$(sfq3RVrn@{%^$-AOv*f#YGelc-)KXVpVeCg+xJI8`PyD@x zMWx~X@wVI+%Y6?_1@sJfwPq(c-jnWbTjtWSo^_2z-lmpvzeAQneGijKBlYz=A*ZUb zKf(-ed>R5rv9+5FI*E9DJ5p1u$<9-(^k0-$`oDtuHhF};&6E$AGlZv466?V+>gG;CVXHr@!= zu{dtm*C{bLbi`bMAvhpP^~4d|~Qq zclXfCgDZy}DoBzF)cjYPIE5+Z9k>dOs)XMdjsI*Ykz;EathiCeaTwKgv8aexPg+k1>U{YMUu)Tjj-?B)akMl5Np(F)`LttC8V4P3ICxK0IRPD=`r%uyau?$n&+VcUt zHI$V3MBB1mo>!FPw-ZfCBq7Hw`i2z+&Ge4rgByOaVw|kxJ>K2oo>V-99;PZcL_)@!X zw4>nK%h*MKm~+;62Njz#SUcj)HeRnd^109FH~QxzdCqsk`Jk5N^n>_FL2RAupr-pIwej?k|d~6y0 zNAp*p{6tHU4$gVM#%M5oO1_cbvaXiXSe{PbiK%o$zr@P#5yKkXNQQ zITL^X`{@!c582Hcg7_DPWgEtd=EEHDjpuu4bqJ62Jbt1p!%`9(vs0jxD!A{EpuTt5 z_aC*(PO4rw2?wSkp6a`|>Q4iU(|pV>t9kG8+f=1EN^} zd)bdb?^pqUeGOGR6&#jSyW;lV)Yc~dic74tyvil6o7S^(h|0P3inMTHBpu+{RhAIT zzMPW|ttIZVlaD@&FeL<_QB{sxemVDjBGVoG)Q|hw4yrC{{mq{;bhWv5r@LCjW}W4` zg4YNOj8 zV)7Gz&OZ9Lm=awZ1={MGUDSfNrmWS+niqV*p7JIy!QTr%qZgX( zJ_xtnWG9!h&s;HE9!b=Q&xqB8;uveCs*!5qKe;2CD7EV|owZD~kuR4hxf0*xhL)X8 zCYW%<$a(--w9v5sGxkGzoe*8Y^ovd-nid}Qyp_5FO5ekSD3w?Gmc=I9tjDIPvZ*>9>zHn@3?LINSUgV=HbckwLi*&8<8%+be5BvB8-G#9nj+yrZ3Ku^kE%1F58IG2|KZbJ_3`U4vt& zi}RPR*+&(+1Amiy2Mz&@llCHpi@wDZKq_*ffsq51kNP>H6~2$XR~{9f?@9E$2Y+Z9 z)v$&u%dVq3yHΝrQeb$CtC$kwRB3e^L(kK5BYN%jdWEbAOE%zofZ*fw6Lo1$_jk z(E^&8{5~4TCqGH+f&0}lR*ocW225nBF8A+FF=ie?BUA`}Ru|w6e*e%*Nq?Lo$Qfv; z1x^LLK}wwcVyMdmHbzbqv~roLpB3DUfA)|7fOhGkiRjEQ<#FBch1}D2RSb*0%0rt7c&td56z~?*w5$ zQuGkaFx}_pK+XTOHC6R?A+M?lQ8lqzV~F`sY@Rd2iBh5b2Qrdu6%I#sZ<}RETO}&5 z1cd+fd8%6O2x37=1EgIX2Na(`*e=dN$a3uJG!~Czd}3~fMNUVc;9{+bQ$3WKmEz=W zr-e%5Jv~Kf-j70zr4_y@mT{NT$oOFGW!$1QCt!0UVOxyapop*OS^&gip?eek; z<$bS_{#EZ2k--*-w}u`36374%sT=D|%pnq=kWmk@lT7-VI)jHg07)E-Nu&~(Fmn;w z@&;hkGtDb-b%P#++!R=x#>RIwm+Dun?!AQd_K|IW@5#isMgpQtTf*#=vHN6edsnqiqyy#$gNF%dB3r4J!N}@Umt+&dsArAh(82FN>hckCEBW{W<&b9i*xoAeWcTb%mM{=E>%NMb){{h?J4D=! z@^SYcl*<<`Q(C2nF}?GYpU&K(thl1H`*li|H*xe}#f z(Sc$X(YA%AvtViE%NU=pJC4qodunLDaclk8xIGE=wOOli0FI5hLu5%I8RtbkVKsbe z^NEC+O;6tf2|eKp%fILpL*>>%45I~qqG9sIhfd{e?(14<7&hg zu^(XE2=PFCFL_4-ZC*h~?!LlIcg!Cc!Z6Utb$HPT1`TfuPl6^+r?(T@0?E}tj^91F zM3@Xm_(YFsuFWxGVTeYYTl@B1QEJNUuuNsI_L zJubyKwc$AKk(r#f6uEGjqVvVb^eJLt3M1PM<%$xJ`54>a?FK!e-R)PDozC`}8|J$M zmhFnf8iGT`)y*fpDz4z=*|vu%Iv;I$|1i+++}9<2zFCH`)pDj@+AppDK= zBW~B0Vb%Pf^MV zSjagj_j^nKO1)7XmThZ)*cIt@pV?Ll_-SHeoQo1g`0^7RiuhV7^WI`z>odgy2^;nj zBPGtgP*pf$b()JRvFI))Bkwv4Y3Dwi2~bk^-sv#0u<1EC4Mvmfrw$3U=`2o;dq|vl z9hNu2>Sco$3y@`Zw9x_9hf#nsj&BHuG=_D^A1%C1PN~rCoCu`Hp5a&D7>{3x0fKC7 z6O@$i%-jDup#a|rJShG*g7O8!Ey%5V@qenPN+MaAqDQs;(p_q$VM8)T@j68H z&JB$pqaRSfZfp-$59FxEkaJ1~T%yo{QdOWO-a`16&t)=fbd{&K9V^7(fdbGiQsUi3GEE8*pcn#?KkLLMzIgfI^*=*!VimCM)7; zXLyqwEsvVO>2C*XJ&D*~>PhyFyTusmy{e>;^^YCb=0m8A-24S73yZc{4TXrE7zZy2 zn*p6*s;m}%jtUFR!C%4me04oaYOgL64$7f;f6Q?T2v>lAZf}8&=WkKUAL@C_Q;PIG zD5jI5htXe0R}!Vkh7&We0&B(GD0AzDHLLo1@dnKiVrRw#x4^V1f9pksJHLL6<&yC| z<}|WT*3$L)6Clo%LsV#JYezIX90YZ)2j~@)g2Fz9lCV6g7URr>-@*~j-}Ym)xD%YW zohXZi_<8%jBR0bVgfnW*!(YR>If&A6eT9+{aGcnS;^AmloXufGE5<;)1f=QqqVP-p zjS2nvLTKk=!R~No75muFJ1F3zyy3WKgNEGb%BB;0v5xfC^9?>?4_2L?J>wkTTS2YD z7+c=daqV*ybh`dyZ}+S)K2Q6Dv9I)TS8SJOp`}|HzPRf-O0~k^btrmvgQT)FK~%3$ z0wQqg3aRU?%hHPuz5G%#i@@N9iGgnd-@y%NgLANdR+5Pe%@)&wJ3?W&MdE(Y7U3_$bfPx!D;%{%o@=iZH%yEAg~)5RpBx(}#(fSI`(e z!TF9z`e}!SSC67FQ^PEE2~wGlP4%AboIV_h2Z)LAZ%PW76w7LQWBkNqYj!3dtOeHA zic_gDbjW(UbH&YZfukk6qk1=Wp59!KqpvbRF#c zDP|v*;gP+0thRb@#H_R9GxiQ0<0qyMkB!TGiHOf;l|8A*A1Qu>v(}~OD5m{UJ)u33 zmw4Tq0znp7;UKYHPbtiiy-*XOkuRVtdDU))%ZI4?Wh!{|=Y{!bUOmCoA(5}+$rjT) zOa*3ico=XQTRy&reqDDU=1C>%YMOJ*XLz2Jt<_KBOFhD_zBhMIu599N;lz&`Yw!}l zzu_%l$G?Ts$N6h)IbXOa-_(6w306kE@;8i4voY5+YA@Dw*t;9nhsn?f<4;iO^#0b||NwD3>$bUPP!j)@Xg|4;0^|qXD;GlCCGOb&D7FHg#1jJ8$S7@WL zOv78p`h37*dGk7~aTsSi@75I%(XHQ?|dlg#5W)zm`D|gX`h7 z%L;cM&&xc|w1>Oh`4Qz3|I>&|V)GAJ=QLr8Q{HiZ0FPH>Ttu#Mi^ySg&*o5{idb^% z#k)6{$6J}Mk%P8W?>;^SdzKr&V>+m2Z05^!ULatn5T(F9kmxZ-3t&!))^>gWD&R8% z+7nlUnMWLc2Z)Twy*_hyNmtFHoJH~LcsW#yu5m>z?6E$bHwkU}Y-rldH?X)kHoVKU ztyx_;pDEebP%M-)HTAeV0>L8KG*dTC*Tu_;v^JE1x3g?!(Jlg32 zGz5Y&+AR4B6*11%`}+GfYWQreJ)$vK^B94?FQ>%Hnf*p%U8`ft=BS0#c3IFuhGEfQ zeok`JE?B(n!b0Bp!u^ogEH!lW)^|Izf>qwOQsYNbVgAZ|P0D{yQzMc8 zHh-GN!;^D=sUnSVzuH4=ocjI7N+P&@JsM09)dtyx1|b`$KNH0n)j2u#&O-WdbE^j^ z06AnszvIj|)(l~tBj1;3xkmP2=-DqBE-)gIjrM%z`UP& zo?Uae&WBOM?VEPTF38Pi13(8xzs43mVMQIM`tt@jid%^&vgDS?xXn4N5E{(}HY1yO zKW_u5_j*ROc?}7g9`%Q``qCADs-m8HAK4T*QR696^BG&ntgCk}-`p6G}@)G@8JNd zkZepfsgp)A>H|LyM`f<<`Z#PkkkVW_{PI*sFr0aAEL$+2ENQ3a&Lh|v86h;ZqAP4A zf<|S0_UkF7VbNpQ?%lvR)H%3LzpQ$LoD#8ctNoez%#Ci{u4O`J^RP3NUM?&~4C2l* z*Eq>Q+qy=s!Z>5wMFkOutbK*{Hct$Zy1??fJrDvmKpT!#L>@~9@LHcfo>R@v!GAHX zZiQ)p-Zw5TF0BVgzvdHrpU=zeh@Yx*;R2e!X)zi+H^VVSHfn#s z{)k^Oj+wC={vj(E3x^bAp5Wo;K+ZLnf4N~c9GD;1>>zY1ioi6@?=#IxfO^YlFbZpRbF$&{Vv;7E3;Z_jh_mb`V$qKSwb<02)ozAnZ9hL@1Unv9kVX=xK%`+GP7TD)GUdaFQ5oc&8YMQzf4uzX0xbv@q7~GTqI43Jyz}2d-J>> zAz#q9`$G2M=^jusEKq6=&W%q)%=~@(HE`e8M8oxFq{H^4f0q|(KF3EmU>?`zM&GD(pc=P^VeZ9c+AE~Ob|CIJnW)0ad~6*5;Q`*GsN)|j1->JK$QDrA zIRI$&!jSCNq`IlTXWR}~aDVFh3m#6K9YM1cR}kxSS}EFR3n$leWXdmR1y{z!+(IYD zW@%Gkv&oWFx92L>VK!Uz0iFHu55hq>0Bl8x;Z7`DAS&dm21UYTtcV7Rm%@>|ygmQG z$dpB7e+^Xq2md_!@z$LYfKriBkU}y7zvKoGNo6WS9Z2u7OONNLa@x+iOxmh-UC-+d z5RbzB6#;it*G2dVn2Dhp^n1r=ua2{`5=UEiLq_qH!tFK3FX@`fTEDuQ)>`-21pHse zcLD-SDvz4FXCg)iVf1UHpn8=tid;$5G1(Jf?29;o}Wmf3hb72a_#iwmubSJM~>M(>txn>Om1lGFd4*60~+; z6U(HUV(AOrfi|=sPHBBUX^AVhr(Q=lwbIKeZP7~bQbT6Px8NlY+HDrJ&qtJ&erMpg zsHsEaFd9Oe963_(_1h3?Bqx^^+0Zu%4dwnsBzn0vDO~+M`FXihCB5?#W}3&p^@I&* zj!f7x>Fz(IlC}64)@m8bGzaQJYMC(K_fx-t6HBaXht8vv&|D13Yoocqwq!YM} zB?@>`h7e;94k(G~H+pZr4s6(v5@Ec@BwtVKblGU+2X@*Z==nhgTk^Am;xOaJmR>CQJoGSz|~mU@st zFxvg3n?T3q=X^sByn=14E`jo_(J!k!_a=4+Wkl4WVFP4>{Z9lT(=fgR1wQ<{3$aCX zy*p>FS(neouxfFc04s@Pf7o>&_w~XvNOaREKv3D28#bllXj$sF@dMnUHXR@$at-d} z0z|S(A0e+?qtJh3FEMrS(af|KGR;F5Yn;cv)wkE#P-VGU~syzls)?q6_uDabUwEc7LFa+D5uS%~MNAFb5E#X& zkhxmS$^o#?26b03qBnU{eEfBzAtb9K=>|@31JB@ zEd9goZF6GePU(O)?IYQ{SX&#&m$<3UmarVh*2nk9`Kic;s(W-&hi006XQzFV(TY-s zj?brrIvxo)CH1k?9VDD}YihsVB*uk{5ADUNydh$TO77u@3vV)}u;58o^f;=UBSz`H zpD^dP1!dOtg9%sOaZ9Io-Mkm_=XiPd?cFQ+A9*v!r5TqMQLUG|U+7bblGC|2pRiR(8g?!oavEJwI!*N-#PJk+zhgXT?B%_B&g zcZ^F~xyZROhn-J6JLK=oA^u=9>2nj)I3YOz)XwG#D>xo*5vsf@y}suc=ApE^jjU58 zWKOv~9XAv&XpLXtCYTZR?;)*gt_A^ZVm2pr={On)WOl8J4V4*9dS=v}B zv5m*%CH1uoaoq+(Fid;Ke@FKrue}c@8L|nYm+HdzJ;K!RNeDX>J7|{FcCMl2G#${b z_c8oXZ|=c*Q!8@V`}Qq<+M)7P+i@9X2j!*x67SAbyh)6Y<#HhJ=|5u}x=`g*47a7V z92!b%>!*|qNc3hbRj`&bRp8uyEodT&{@}4j#yx#9uCp23sg0p*fJE6-lCCmEJ|v1* zJ7$e8^D?C$E&EjaOg(cmXOxTGF7`qdVs{_ky`i!{gIRh4gHr>Zr;bg80qM|vb~pPG z1o_yalbga=3q`&(ks&OC%JWN%>q{p*{b_uty-YsXy;|C$=B?szp1oyPw(~x?0C83c zA9bD_Si^`bNH5|{)C%~+s|=f_V4+G39`vWm$*#6HU%D7cr zNn9c<5jA-mZF~{Ukb$To}4YjXZ*2#QDH& zM&``@OZpea^&!jTV9uJ0N$#;Az8rog(H}E!AF$i6XQYI9t_Ci3KF`I`WIm3>-zBF^ z<~hs7%yN?tFHmE<4s6i>Ml_K<1uL7=CH8yom)3q7hxIJuz#?@eaY_!x%*)(_+-TyFE9q1cjW85h4@MO&P+a5Qd7wwq3tz6#WX_hL-?{j20&DpuJ(Y0bN^F={GPzNh0Z zK>-m>6h2my>}g2|YuAqBA1lsC@IGzkX=L53B=99!qL`T@$+M z>&)xCviYi~-+{a>3aQ%Xm(>f>UcN||*HvFIX~4+{90S~YVVqq^)%SA3o4l9ZxNQ(C z3`l0LC_C)0nWetKeslBBdFcUk=F^1YC~%{t`YT8kgVk+k12(S&dTr9pQFJ-MJ-U;p z?BbZtQ`v#$zDJeDhDMG=(@feo7=uwacm)l^=6B*4D$f-cCAMoa`(D6cYU$0dY+sob zoH<>tik?OStO{J{qYg5b;7=9b7rhhLKv@+9iIM)&~(Q z4|mrWJPFn3zVWx}nKuudT;8K|B^H$?-HW7p>#t|1Y0}Qi0VyblO+qEVe3OuuYfr05haj25%T{96necr*?F&F8eKJ|F5fe10 zHym{O2PpLLHx9aQbVb?f4fUN(^~Q!vSNvtK-T3uk7QWyO5^ucLet)*QXpK){tbm=P zVZWYwp3Vo=A_e9N6{#cq0&T+UJuY!JqA6pgC?7yl&anZ*bmk<@?h>&!A;zRJjkw^N zFN$9(F-LM`5j*zJBNTxTR6Q2nS#K|JO3TJI7Mx) zYT_pR00ynVYaxA@{ry}eAMJR$E>-$2;iB# z{E3KtLaCCtx>pxP>Mc*ry3jgqYo6Z=SXDKiK4c3IMD{g@O7n`B7vi(oQ^P{QcT)~P z{VChe;UF+mgg%8^S~~(@dX!?UJezH;3*`Du$5Uu*ic-;6S@&*HD!Sl5dDJ zREI_HE<5E@=zekeZ8j%5!?f|&i}WPP6;lC-e5dYb`QmbGKiZKr(l7niikd04egK1y zzUQ(sIpb{>w)M6H^&!BB`X2Jcq2AhNa^{wECTxc?Uq*uj1Oijr`TG5ML!*^5?qFg_ zueYjQArA6StupHu6gx&~9M)}Q@P#?nh0{;y6FRMY6IbR5Bt|u4{X<5NxmOe7ZvuP5 zAdOwM3i1%Ax!)#IPYHZL37AWU?+U5RThG{H^Y>#>cr$POrgDIa1s_)WuIe4IzH1kz zYB;gfT@w%ka}2YFESci2YV3X(SZpF=*|LP&Me!r|@mE|4%-$XXdRprIHLRakUJL5m zqi=P~#3@FN^Ci76lp4E`%_RK`oY%S<9~4xf|Gn;$9%wI=GInYD*`+P|`eiBISbazd zs$TzXPNTb;Z5~L{lb(-+{AaOoWYm<)K3E=L_G5oMG0N>ymE*8Mj>T)anzXRg^|fK7 z(|Zp}<|GncS4kU^jgKdD-ZE=_y@%|niM`UIKrZa1W#kw5xMMxA&(ZLEA!dv@==F24 z8NIcY$KLnF0P3s1)%;wCRxo~$kr#lnR@~iI(8L^r?9)*k4(=MN{LNeh_Aw#iWtXJ6DK_ zi@oOI<-g~9u1FdmN&nJ|-`>W9)?HpIvY=>5*#)V;!`!wsE9hSsOjwag38wJ5gvXWH zjwu=$n#N8wri%-2Z#m+pg9<{TgvF_5U))%KxM-%xcY9-V*8W@!jnBX5xwzcdK}jjH zj3=oZySoRFm+C#MIrgwHHR==j^^t?tOQA4K7RdW@B9|*;eaa2Z5#u{mTivC((q$`s zA#3Ow|A8%74;{I2GDL5|5|hf|LSm$Yeb5)MO7Z{r3xv}(aZam+`%|&#!p?tBYHybO z0?V&3txq0bcqjmXgt%?ud@81ykYBrCK~g_fC0>`T={5~LePT7XzI~V7c(`}hbt5Ug zyG_YNN+%Uty=AMUC?T^y9P8(G*mz8vA!dIR3l}-0XcRO=pQ;we@n9FT64NE|@Cwm4g~wuH zU~yJOFwV_Z$B{{!rK>YCa^>S1jlc#W+aM{ub)noML$mKkpY+3jF^Q~)o zlcMu6PUd`%bY#3YHYmCCQ6E3R+CQ&*QZ7D4vg0d)ZJxU8D;W!TKn;c4-dhLzS?t#^A0_F7ANtpt*k3V;Ce4Io5(lE)@d7uGKB zcW-7Pg%D?yFzxBdYAe>2k+cJ-Su^&RdP&rzOd(6Y$?t|rmXAtiAT{kpT{4^Z_RPG_ z7noZU^t%Y;XAAHTToC|wQQvSZQdiCeO70bBTjdZB83B2w z*R&5@c-lM4RhB+gf--l>=!t2ZHG(lJ%P;eu<>cz*>v|1J+|gVYIushfp5LE8*l@Ap zFX#^x&&!mPuZTUMCt2gj@n-63Mj=t2i}^xGU@RsB^_l_&-%+-d|D73Hrp|FBL+_OM zwT*Z*@7&TM2T>08ME}K~Tjq;mY3&!FLq@Ll-cLj)e3z9s06R$h?QMYju37hM!}leY z1?l)%apjFnUkpOox43vM7Vo_372@b=Xk=u*Anq9|=el6k-n>1z7f1$CZ3AN{OO^9s z@b3~gOkM}pU2d5fb)gYT=Z44ZMOP9!_=r+LnPgXrOVWkSCqp2-EhCcUVsrCK>f=z8 zV`W^{h@m+w9m+I*4g0-t53xdFB=(>CcQPJ4xDNC4at51* zlvxzku8f0tBlnqfQI~S)DSgqubXqvz&oBSwf9+ed&6I6aT^P5y^2ddrPx_mm`G}B{ zv;V$Z4mRgs?WFLl1Ha#esBz&pg#nCCIP>M{>_wb!U5;=TjsjICNh1ev;AwADBWfs@ zh3ZopE%vGw*_bw2kg-Z3^^<+jG)v4B8ia;mlZok9LB$Y;bVC^NjeF*}2TN146IbWo zv(W&dC(3l`dN~$3)(>ba*pZEM{)irv$V&fy2JaRQRfWg?+PPOMd4~3)DINHW?r24^ zrpCSOKR}qH_Zb+OXTdc2d~J-@z&Y(-MvNAGzdg_xc{amCWi8lOSbgd4XUISgYlD$gief>v*Xs2K%&m;dNv0CIK?Nif$kvqE^OcF(fxW0 z_~|b(K~G*N*@&x9fC^&e66Q;GQv;?yl8LdA0SiGO-vYMj`Z@FuU;O9KU{oduNE74w z;H(tt^BjPzB_qVd6q+$B z{PlUafPhcsboe*6=9*$~0bXUl@EQ$48zsHfp@=aW&X5t%$T2gZl$MAy_spRzj@?MZXfnSM1mO}Tc5&t4$O3)9VUZrE!nVJ; zWVkP0vu|ftnsZM$qaSs&4KAL{e6avK)8;^a9M^LOE_*Oc>Ujb#`~|U z-U zm4Th%Ad7*sKyqcE9JmDnJ`rh0%Kq}NYy-nPrA(JK`MstGx{s>3+U9T#@rQh}5iKs0 zkVD~n-LZWBQ6?-SOiOi0Xb5|qpxN5>XIj|WEf+hEg@V57@Ct$QyL*}6&RP!`yi5Mw zpQ#Q0(yrr&n_kOAcCzRSCu;XVAL2a67;BB=HzaRiA5Eqkvlzl!5GI_<70k|VV|_aZ zqvWkoC$Wvtin<`>sY&KyjU5E=QD%-XzKYRUeYsGf3Oy72ttCiAH4@$)0@M-2b3CE6@(n$D5KYr<5Z~Uek9cE64bYu@rI*aU zy_&l9sTtQ%6@?cV{eSvl{qU$p#ufYZRJ*y!*E?PXk&)+YJJ-09D2HB=ptH~04VtVO zyeY#-+ZOn$yRas20hU|>Cm*LPd?Uotf+m7Oy=pF4rZE~ZpR&bvUgu@E5uFr=H zPA-|u@=>=o;3WdI13Gf>pSwn;#kKw|;*|0vN_>ywy$7*Uik1fUs{#8jDHMslp!)UU z_c)D9QUaG7W~zXOw33<{xV1ULQG`bU;-nk6mlF>tjHg9rf>LH|19wqu8U3|iP^%!H zg%K8szsi zJm+Ag_~_4%ck+M=bcPn~pYt`XE6rss!o?>}UqKkxzc zj^UQ9+HG^Q|9&7V*~6rZdAm#f%0wYV_KtXqj(Yb-bKBMUr)j~vIabTPeGM@_t`C34 zrZ#aA%IC5Qkg9cfZlIB7KOH?+8qRP8!Sh1&v0Uy{%EZKNNPp}nXt3)goua))N#GXC zW>L)r1(HLf%Mo9cWp^?wV%X*&o*woEK87;Mn?D^4?YvEt^_9(sNe?J%)hJ_{wr}F9 zQhl^1VF96=Hx{KVpug3aLz!(dh{n}R={K%a`jT_2WXi$ML2|BY;(Z0%xA zUuRWfV!v^;=Fs;OIv3W6w?)Q(GdO2V--9&+Guxo?Q^Ux1gaBQpIWGL{kiZh%m#i_| z=IrTG1%cw!vkM(q!j;8aaHrHRWQ2m zNmOKpv%Fm$lxs>Y+`8}Hec;g8(f>%FQp+lk8jhm&)`fgmMML?FvUe_r8FKh9ON77#H3Y<0gQY1(@57jsQdXM|zJlixL zpk#rOs;5ZgLC0#-t6G*wpC+_JoSiOLuQ!|_GJ0{w)|4|zsC?^xhcd-=$V5i2^Y+Nh zWL`1nJbpI5j#XBhI1}@`@*fTR4$H7pt%PFshxdgzLfR5o?{xEDo3y59Z+9>QlxUb|L-53t^wDpP7MQ9=_)y6z0dkmzg`Kd z{JwPANSXpgkkCBq{$Nayi-oc?Kdq!|)`7RApDCv(#a+5NtvZ0v2wrjMM)<^IQ$;`K z^v*HXrdsF~Exx5xW497)VOgI&gmEUkx@M15$#aXI7pFU8HSQ4HRPt%LJBXkL`}AxE0sqK!EU8?j z2J^YnA?cxA5z%9i7BDtkA-)&$5U}wH*sYrTACbXJPUo4}nkL ztKps8<06gIRKq#6;L79S-P#0j{e^O<%l>BOmO0Gz&{%GIyi}B9L0OI(<@6aI5oya8 zE_>;WicVQk5xpQX#g6y8TdFhqmuya|eG>tBOn%6X^A%?F>C?nW2QveCjIafCbpqFI z_4Z-rI+L_Cd^B`5x3qFs!zZu9Y%fnSVSmW34Xx)pPaPM-@-J9N7`!kN^N9~MpR}t$ zTafY+`+w`O=+*@FWWN1^EF!)1TF9^4bZ=5wIB@oHaYjL-_v!%>d?Ibrx|`*8?x=R7 zD6!QlHbp^7OfmH@bJ|5XzA2cAw+~ZQE~TGJ6yYOGYmswU`PzMD^T`j>Cut6L%EBqH zh{NOfQ1!r9&}-Iu4PmL_)Ul2KU2hf}O#uPjP&#)@$Dr?RESs@f#@!4{MZrvWcf%@_ zw6)cDs+2f$S!P;uwQ7+KdJlK4xadhTllueM*+@`kH6;&tH;;>|wcd{vQ1oC?;@^OmI~eCEl^oWLl& z6^-=DzDZy&E7pk)O*hkDGHnu#x7t2F&z(W?C(k|X6YS(whawzC0PE~}&YdAL_>C-N z>^fL%7t8HtZ342}8uO=BY>;crBs=@l$5DeV5xU>T|6}s=?)sQ^A1eK89sCCA@92}n z%1Vpy+trRwy0qb7E52C<+U2&t2ML{~`}47cQWDc`p@`+o|(~ z%v7HT`}lD06!TV5DFn8aX!wMbxV$*tZ;fKw~|DQ1xp;Y2*bT7-RJQUd$hd*=a>~Wf1_J#2p-sy zCEtTO@_ao>| zS_2=T9WhS$xk{;$QfIS}E*Z+IRI;Cp-PjmMiw$98!)52dRD8}>l8g-mV+C0ShFL23 zU0(dPjbBdzW#TfYM6`gGmy-PmDWb-rGt7x>&HVoOG13mP&9^BCyAtt3P;0W@jtJa_ zWd@$jZ2vt#;@c|CNxV}JT12dQjvG^-^|WkXp?%+ZueO}zBwlx_{Ycj~M=cN-DiS17NOvJ=8q zxbkX;XJ-e0SZieU9y{&Fy*;7F>-0NK(=?5Pe>L&9#$PlBsMCrCk`)}H!91fm$DqR} z^=A>&amBk;^NSvwQXj@ohUEjhbV7S05F2J8U=U`%xuTTrVwz@SgxwJncy&ueq1IPlZWz zHZ!nbi3#y$*!jhw5B!OLkNJZ>hpJRBYbrNj~}i{55@znZWFo_O-5PH3zADw-Jq09$VooTk{Tjb>nw>YWZiT^_*@ zpX)?QQfX}{neItltUG$3%sYb!ALjkWisshuy6(3XYYx#nFcBPo-bP5{Q$1BIrA~yd zu>WyT`SYurDrV=d>PyD9&7=s~yvTnAh^3Sc6HLG*Z?@VAF*YxTt zbHmQ};(u$xoa6kc%DFf&W4^xLt4nVCU)<1}L`kat0pt>+XDPbGW71tIUOBMo~v$^rjQ_++wBH2IH)ks#gnR8L`bQdAww@o z$}POH6R!J?pc|l$U@Y^xaV&<;y^%=V5=!4sD7RC+oOwk#Zi0g5dmT^nM1j2q(3|Vn zUKZ|qX^8Jk10&-bz^xVM=hq&=D>;stMBhp>EyYxh_-C!C{*F1E0bK!rZjM!H{x1a7 zSpWW-s4SE#PP3b7jw@k6M?Kjh?hWD|qaQtn1zMbozv=BnK%wgiAZJlUyHNbR+l5}( zkfBXpRXiBJHAQ*F6F>aYnh2&?zhQvvW#{H|H|ID@Ob@kY(R^=%O|)Tr@kBYE0hKJD z7X01M3{Ghc;bIc$*|Xj}B!$SEXH$Rv{MuarEr#K;U`Nx;Fly@J02r7Yn*|_so;&tq zm`mJZtab&OuphnWa{d5}?F~a}anR`(VcEA!Hb{wC3pLFb_4)Uxr^f{nROES5eF093 zp6-G`8G-DlaDa1=fpRdi-xi^xakV%8FK$oF5ExW@VT5G(rNFxURlhsgkNis#kJq`Z z@}5PTXQ}T_Vl9#W5-B6$p$li%Ro@H}yZXef?n4)#ba=(T&o!z;BOJdo{8%jNTBDk0 zMB2Y2jQSpC=gFYkk=^y%KH;fG^Zx!CS$7{DOz9^+ z-mVs$;PYcc&yyRxElEJBQ4je&rk>LCtXxF;2)Ih+-2Z_ZY)ILCm3U)qKnB~el`?4G z3Ru6Rz;@lwclX_e95qwf_0m2N;?*3XTc)J9IZa`OBpWz;=VJzR)eM3{0JociJnqq9 zjMW&8oCMu}6#h${};M4&a5}qTSKaCkQXh+It!m3m(adxf|CgTF(EZ(4|Fs3Mp`ox8zgH9xx-`E2n%PsLz`C*tq%&Vd_VxZ-QRC zk4*-%K+U1DTe_%;R@i{M%_((|yXob>DvB~!$;l_+s-x|T1Q`?}DXjo(o@jPtfDOEQ z`xR5b6BT}Gc(FE1(SUp-gGO|ORCpc&DLp|DH#@PN+yT z{vZxDQ5W%%{Jqe+32K1}zY%g0y?#=zL911!PbKtS;J2QtVH~{VIl?p90D?$iETV$Q zPb3-Hz?X!O(DK$ai1e;+!Q304L;3Om$v3TQOS@|3=J+MC&ZkhS-TbdIpTFYLYZj*) zM{ZG{*a)@+43SOf*UoA1(wl&)7ajNt5{U(tkrk~wQUOPa{!AF3#is1mnA1x%(lFW2 zChM{%$38rTb^~pm<@YaapnAY$fOf3MuIa+s%Uz42xF0~g2d+=Yd^cl$@XEWHkNz4B zTH^1z7(-K4Ib@_9c(!*zi}}&py9XoQeW!dL$DNH9dw`MM`H>hGUKOwn1k~4;lw%M3 z+J8g1e{VAkddUK-T*!tpbe;1xVuP728z`Z=-e3ALa(~{9>O31^TJ+qG%j>s4fiCTO_~Y$a6=)QpX!^?EM- zn3O~bkq-5h(eK2W`d_p`Mr)=^emEQ&AODTqXC$L&?io-p_8=RYq3)s_RT8gxaQj** zLGVa;V;x2NL63ODSL5oEXC^@elL3ZW=%40Ekc~h97CI=}kZlDEA zt6F65)NRDo%3O;S>!8X;i(%kR1b--rtJI0}1GML~(YeRv%bVg`UF&QgKkR;X=+S|* z|BD>>>M}(-9I8=@)XSPD|IYfJ?7Vw$m?(N%J%+`+s{s8{4B2n@?7Ukw)KU4vpZy4! zID1g>@4=_F0dT(2(u2|S^@I4e9P`7S$@>3}Ecv!kX((Sq*c2Q&pWp(Tf&Bk%qy(k; z9n=|j!2e(M=4AkamR_XmfF2>UVQcyMt&V9@wPEdWKwDZ1wcN)NE#RQTIAa#j#0Zen z{ukW;9ab|8D{2Z*Cg@KjN)7z@)bcdfWC-)WYz%C3aMo{=t#Sn4+qiByXc@}ySV6Mu zK3wwJGW0pru=?|EOjnWdJp`=Kg6;?rRi>Kn7}MBB{jBV6xVSDCk{a2B9iH_RebLOl z?f>s(XUbzv8FB;iniD^vYr`4v9zbX|A>1R`kINl%%ZNkRGr9|A0bYM4q7m0tygb1eBQm<6!wM z*8K_agd7n!Kxq@5sB?hKvj1OuR~k=c-i9s3DbzVya4d0%Q)A2&UQruUmOKO3P#a#8_~AzwpDy{hg>7RVE|>BH4my^beo^`JAv5CNW$* zy1?BCz0106Q9hy$nV~1J>-zQ^n-Jo^lFcJ6+)nmDrXeBf2xFeP7BX4I=no7^C|tN>Rn`^?~9296*E?P z;UF5TwDVmRzjO~61j*m^CCXW**l+piAq_w=L(NWEmBfo&AZyZxBGO-ywg}km>-0JO z9i`fbe^s+&9Hw2*$KVgtKn*Z=rMY`7CYrt9m=qaO3VPh2J`^x^;3S#nSX{l0F@E{l z<kJW%XnXc$Ait0)=aAaRe( z7L7_Wsg1W|Lm_5J`&Abs&k>XdkOrGlyWqtA5XLt@UNU%Z8ENi5H@29=JiU*UNj_!! zkw)`^5#bUhwJ{3c0(jirs>|R7&<|FJEZdhA_Q{jtF_?-ea@APtp-V%}sQ2HG!oCL` zQAaez65m(~8w3nTnHZD+jt^jin|4svbOa=~J#^ zrmJS`pfvTDygglXyzap(oN>9e+~zHii&XsKZ=4M;Bo5NOt^j`AnO%pB_ZNC>oW>ZY ze{I=Z{Y*tu|;qK!>9=Su%FzBztt;$wt6CuYp(tC5MV1>EZoLLLKmg| ztfFazMg);HQ+f}dTu~Q_d7rM}7XFd-yT0K8{hNS>>DE(3J_A0mu3UH=Sf| z5?NHH!QX(=Jpb+Rt^Pmd7iUa!=N5nm!kXUB?*%F?1zLWIINr5y)d zwPE!gqWt?ndwoOaQm^Y)pI9x?-~sa3`2)P_Gnm4-=MnkIX`ld=USj;UsnYr&WuF(o zj4uEV%!5~TK?8DQ0MvmwLZPo3)cI*3oK*Fi?Ad~{AZ>eG`b#9_wd15ok1|j!_$iuX zL`TRT;BZAvX!irfkI4M=rC4EEF>mFSCtjdhVd!_1ggB;-s2CLkrFpJpaeN41gB>PI zj|>NtSAdiAs%SHF_Qmh6=cYp8s0IC@fl)ZYa@Rv}t~I`pOU(OYwM#uBpCegD?!N9D zD4i%YH$Na~%I{M!Z@v@45OwHVDF@b6DwRPA0gAJ zn^wy;5%<@NI2T4}bg*a#i0^Dg@F&Q!Ju(@|f<90>m)giZyZ(6FRfGtX!7uue&%{-- zpKdMO<5ZK zi)y(U;LZ7Q1Z7RRjSHknXc;JiE|bPKQexH36J-A8Zx98n;IyiGi!qg9jrW@J9v#4I z?P0iSxPY^~FoH%xzdGt75$MF06*;m83Tr~jsD~E{yi>}oIk@-?2%erXHZ#;1!Y$&;nun2mY9g;m>XfvhJJq86qPafV1nzAU z;){0h!$9NtKG4LXbz@{xGPdKk8&OF_nst`)^oCFSfnQ#E0-MHg!f~RZLbxnuBbNlz zj%urLk!y3Ou4tDeIOfyfZpGrcyZmux;=1us+f+&<%rX~ris%W)FP^X~_Mj}nvB7dh z6es7VR%~GnC(Js}`P>j3n;p&x>F6os0i-?Jln05CbG_S!rtSm66K|0ba$-f{z2V5>kguQd!a;C00ueC#+?LhT}>;t zQI1<}V{v*_)(X<7*xejs5ef8oc2nT_O((OU1dj=}{0OaG>j-xSL%GW?dla4UhGb^^ z-4e+d=5frqTN93#^woTX1v0ZlGzXE(CIWvRci3*v7MWT%_fVU&4H9k$hb z!pRBk27Diz^F4%eeNCrlQa|1$i^WJ68t2u_8^(a`2GQHkxTby}Ihym#J#^?oz_lCB zKW2x#L!MhXsewr+mzvux+qv(y`HCX9lVt3`fQn5VBteKfO_eK?Ge9DdtmU*SCbU z9Uyq08EeEL+(L{=x-5i<);5Vv@kHiDp0~EzUkSmV_ccMPh*;eCV68S|PFk95)&-Sx zZYB&L3dT;+%jA#lX3CC19g{a%Pq8W8Ct&21SWQ;ZVm2wiQE$yNApp&J3%FIV0>1|7ngPr`79+ZlQd=Qig-7$gU~ zj;?@?xFnt^4SONvX`=ZT)++wR5ui~h;>-g{$k;Am5+(cK+rHkF9dZNm@75x7|3^Ra z?tNH~xqGy2F^LdxUX=v|BvPoQ#t|+O9;n+g!!$R&+hnRXPVDXLC)osENrs$7@F|ps z|6wbZW$nV_U{=Tak=g#$O5WHe$6{jpQ&Ofig8tsbtR`Ya9GY_&F@za zc84xxYJGvyP*jAUg^Lk<1X#)O_tkDsF>qPFp9~shtw^ZYBNpk4t)1ltn1v2%g7Ulb zfYZ7hQdD88f)OY27V91_issAG6g({`1I0j`NObmz)D?862}1tHW~?yCQjoofl4}YS zNlFI?ob(=k-4@3=dGrW|J8}r#1?;Q%QhfChkQyBO%cVQ0#fgL8A$Vies5|#2;Bf_x&a&R8h0C^@HAI?v)fzuY-2Mnc{^(1JVaXc=3$l9sv~6z0lxgf zo#;-jE{@*`t8xS`ZAX;)m`m^wjzahAOrlKO;z)w0;_tU5Jy=o>w0f@(9kp(2e1>jQ z&!rnr2S-m+yCWV)>byh#X^Z+Yg(8>)qFwzg+#=K*#C1EgmYRT9utK9bZJ2f-f?U4x zr9lIx?VvA#Fz}@Mz-iEIakNji7Lpkwd-%rUxhv%}ZIsz$)ZUi;lCJBO1AlsNRZ-r+ zXCcDN_gp>?a}P1Ms% zf``tLGB8_c9Eb1Va4>a2WG`lAmdpBxX*t5xGMy`WPLD*Bs2%XrM`yd8*X79~xv z1q7T|f)Fi}EP*4mf#Op1->=ZAk+9&6%$TFkfA_OjZ4s@Lx^!g?l7H1mH!z*OfXW`2 zM8-Hc=^%U)WDe<>0(T8~Tx1s4|aqT1?;>{P9sv1NewDJI_8EA@B=lSVw`G;GU?i0E8WF}_RT?4Sf z^2%wVAuIb9c+lQkC9q^NYqr*hcuOxp_JqCAgO}`pB5-b}#uwP4bb^RKIueb`u8i%o zMU^$svlh9sPG1#-y!JXFHL=4!NZtgP^#(rYfq>?s{bW*T^4UJ{q@*ca_+T5fwRLBB zrqnKY?RP(4{?QQ+&R?jNkH7cSLJB7EstD^WjFwrKkCtJZHO93Di~7(G_vV?Sxo^Gm z{)zg)km>**7%5!F$jUVlaRtW z(u2cZ_Rs)p9mWQ1aO)?6LIi#TE}tu^kNhgBLJ}g+S@<8#{S@$`3)pQc``iy&iwYIH zj6aaSK3tixJZxTO-AVbED7{25mPG%l=KsZRyd%qWv7qi#KQ$yHLxeaET{`w@i5t{4)95qvUX+k1;y5Q}BJUS-Qe>lcmMzZ literal 0 HcmV?d00001 diff --git a/results/analysis/stability/endpoint_domain_auroc.svg b/results/analysis/stability/endpoint_domain_auroc.svg new file mode 100644 index 0000000..fb52c89 --- /dev/null +++ b/results/analysis/stability/endpoint_domain_auroc.svg @@ -0,0 +1,43 @@ + + +Exploratory AUROC by endpoint/domain and predictor +improve_benchmark +rcc_vaccine_benchmark +zhao_vaccine_benchmark +BigMHC + +0.546 + +0.544 + +0.542 +DeepHLApan + +NA + +0.461 + +0.507 +DeepImmuno-CNN + +NA + +0.472 + +0.526 +MHCflurry + +0.537 + +NA + +0.543 +PRIME + +0.597 + +0.622 + +0.537 +Fixed pretrained scores; descriptive heterogeneity only; endpoint/domain differences are not causal estimates. + diff --git a/results/analysis/stability/endpoint_domain_metadata.csv b/results/analysis/stability/endpoint_domain_metadata.csv new file mode 100644 index 0000000..fac0c3d --- /dev/null +++ b/results/analysis/stability/endpoint_domain_metadata.csv @@ -0,0 +1,4 @@ +clinical_context,dataset,endpoint,n_patients,n_records,n_studies +PD-L1 checkpoint inhibition,improve_benchmark,DNA-barcoded peptide-MHC multimer T-cell recognition,70,17475,3 +personalized_peptide_pulsed_DC_vaccine,zhao_vaccine_benchmark,post_vaccination_IFNG_ELISPOT,352,2315,1 +personalized_RCC_peptide_vaccine,rcc_vaccine_benchmark,post_vaccine_invitro_IFNG_ELISPOT,9,129,1 diff --git a/results/analysis/stability/leave_one_domain_out.csv b/results/analysis/stability/leave_one_domain_out.csv new file mode 100644 index 0000000..6e47def --- /dev/null +++ b/results/analysis/stability/leave_one_domain_out.csv @@ -0,0 +1,19 @@ +analysis_type,dataset,held_out_domain,metric,n,patients,predictor,value +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-basket,AUROC,12115,50,BigMHC,0.510179883078191 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-basket,AUROC,12115,50,MHCflurry,0.5300323841168303 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-basket,AUROC,12115,50,PRIME,0.5741732163043727 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-mUC,AUROC,11240,46,BigMHC,0.5636546281868342 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-mUC,AUROC,11240,46,MHCflurry,0.5404874933634612 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-mUC,AUROC,11240,46,PRIME,0.6117649277495425 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-melanoma,AUROC,11595,44,BigMHC,0.5708696661696276 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-melanoma,AUROC,11595,44,MHCflurry,0.5392223509184932 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-melanoma,AUROC,11595,44,PRIME,0.6088626172526672 +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,BigMHC, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,DeepHLApan, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,DeepImmuno-CNN, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,MHCflurry, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,PRIME, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,BigMHC, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,DeepHLApan, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,DeepImmuno-CNN, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,PRIME, diff --git a/results/analysis/stability/model_selection_first_probability.csv b/results/analysis/stability/model_selection_first_probability.csv new file mode 100644 index 0000000..a2f916d --- /dev/null +++ b/results/analysis/stability/model_selection_first_probability.csv @@ -0,0 +1,13 @@ +dataset,metric,predictor,probability_first,task +improve_benchmark,Recall@5,BigMHC,0.0815,immunogenicity +improve_benchmark,Recall@5,PRIME,0.9185,immunogenicity +improve_benchmark,Recall@5,MHCflurry,1.0,presentation +zhao_vaccine_benchmark,Recall@5,BigMHC,0.001,immunogenicity +zhao_vaccine_benchmark,Recall@5,DeepHLApan,0.0,immunogenicity +zhao_vaccine_benchmark,Recall@5,DeepImmuno-CNN,0.999,immunogenicity +zhao_vaccine_benchmark,Recall@5,PRIME,0.0,immunogenicity +zhao_vaccine_benchmark,Recall@5,MHCflurry,1.0,presentation +rcc_vaccine_benchmark,Recall@5,BigMHC,0.0,immunogenicity +rcc_vaccine_benchmark,Recall@5,DeepHLApan,0.0,immunogenicity +rcc_vaccine_benchmark,Recall@5,DeepImmuno-CNN,0.99,immunogenicity +rcc_vaccine_benchmark,Recall@5,PRIME,0.01,immunogenicity diff --git a/results/analysis/stability/rank_stability.csv b/results/analysis/stability/rank_stability.csv new file mode 100644 index 0000000..0b9c38c --- /dev/null +++ b/results/analysis/stability/rank_stability.csv @@ -0,0 +1,96 @@ +dataset,left,metric,n,right,value +improve_benchmark,BigMHC,record_score_spearman,17475,MHCflurry,0.4318233968048541 +improve_benchmark,BigMHC,patient_Recall@1_difference,17475,MHCflurry,0.009375000000000001 +improve_benchmark,BigMHC,patient_Recall@5_difference,17475,MHCflurry,-0.008574079429342588 +improve_benchmark,BigMHC,patient_Recall@10_difference,17475,MHCflurry,-0.0029136287688919377 +improve_benchmark,BigMHC,patient_Recall@20_difference,17475,MHCflurry,-0.0563035405140668 +improve_benchmark,BigMHC,record_score_spearman,17475,PRIME,0.40658523103185173 +improve_benchmark,BigMHC,patient_Recall@1_difference,17475,PRIME,0.010405274550011393 +improve_benchmark,BigMHC,patient_Recall@5_difference,17475,PRIME,-0.020872129651621482 +improve_benchmark,BigMHC,patient_Recall@10_difference,17475,PRIME,-0.055536084619369014 +improve_benchmark,BigMHC,patient_Recall@20_difference,17475,PRIME,-0.11424413377586748 +improve_benchmark,MHCflurry,record_score_spearman,17475,PRIME,0.6508621316344753 +improve_benchmark,MHCflurry,patient_Recall@1_difference,17475,PRIME,0.0010302745500113918 +improve_benchmark,MHCflurry,patient_Recall@5_difference,17475,PRIME,-0.012298050222278895 +improve_benchmark,MHCflurry,patient_Recall@10_difference,17475,PRIME,-0.052622455850477076 +improve_benchmark,MHCflurry,patient_Recall@20_difference,17475,PRIME,-0.05794059326180068 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,2315,DeepHLApan,-0.03852733305541504 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,2315,DeepHLApan,0.05223555070883315 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,2315,DeepHLApan,0.08995819701926577 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,2315,DeepHLApan,-0.023209741912031956 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,2315,DeepHLApan,0.0 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,1015,DeepImmuno-CNN,0.10932207075221095 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,1015,DeepImmuno-CNN,0.02083333333333337 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,1015,DeepImmuno-CNN,0.029166666666666563 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,2315,MHCflurry,0.44386743408029294 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,2315,MHCflurry,0.06068702290076339 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,2315,MHCflurry,0.04249363867684475 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,2315,MHCflurry,-0.008651399491094058 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,2315,MHCflurry,0.0 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,2310,PRIME,0.3699749982360709 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,2310,PRIME,0.05258397932816539 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,2310,PRIME,0.04298018949181737 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,2310,PRIME,-0.015503875968992165 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,2310,PRIME,0.0 +zhao_vaccine_benchmark,DeepHLApan,record_score_spearman,1015,DeepImmuno-CNN,0.40610422466771756 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,1015,DeepImmuno-CNN,-0.02083333333333326 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,1015,DeepImmuno-CNN,0.012499999999999956 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,DeepHLApan,record_score_spearman,2315,MHCflurry,-0.040780884614182465 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,2315,MHCflurry,0.008451472191930237 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,2315,MHCflurry,-0.047464558342421026 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,2315,MHCflurry,0.014558342420937898 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,2315,MHCflurry,0.0 +zhao_vaccine_benchmark,DeepHLApan,record_score_spearman,2310,PRIME,-0.14011603954761742 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,2310,PRIME,0.007290513104466573 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,2310,PRIME,-0.040620770271933226 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,2310,PRIME,0.008065706902916303 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,2310,PRIME,0.0 +zhao_vaccine_benchmark,DeepImmuno-CNN,record_score_spearman,1015,MHCflurry,0.09367228092520263 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@1_difference,1015,MHCflurry,0.019791666666666652 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@5_difference,1015,MHCflurry,-0.007291666666666585 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@10_difference,1015,MHCflurry,-0.012499999999999956 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@20_difference,1015,MHCflurry,0.0 +zhao_vaccine_benchmark,DeepImmuno-CNN,record_score_spearman,1015,PRIME,-0.030588882476202253 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@1_difference,1015,PRIME,0.006249999999999978 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@5_difference,1015,PRIME,-0.005208333333333148 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@10_difference,1015,PRIME,-0.012499999999999956 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@20_difference,1015,PRIME,0.0 +zhao_vaccine_benchmark,MHCflurry,record_score_spearman,2310,PRIME,0.7408911584698367 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@1_difference,2310,PRIME,-0.009043927648578837 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@5_difference,2310,PRIME,0.0075796726959517224 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@10_difference,2310,PRIME,-0.006718346253229868 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@20_difference,2310,PRIME,0.0 +rcc_vaccine_benchmark,BigMHC,record_score_spearman,128,DeepHLApan,0.24047728053107473 +rcc_vaccine_benchmark,BigMHC,patient_Recall@1_difference,128,DeepHLApan,-0.010101010101010097 +rcc_vaccine_benchmark,BigMHC,patient_Recall@5_difference,128,DeepHLApan,-0.003575436908770213 +rcc_vaccine_benchmark,BigMHC,patient_Recall@10_difference,128,DeepHLApan,-0.010582010582010581 +rcc_vaccine_benchmark,BigMHC,patient_Recall@20_difference,128,DeepHLApan,0.0 +rcc_vaccine_benchmark,BigMHC,record_score_spearman,51,DeepImmuno-CNN,0.3616192718710379 +rcc_vaccine_benchmark,BigMHC,patient_Recall@1_difference,51,DeepImmuno-CNN,-0.028571428571428567 +rcc_vaccine_benchmark,BigMHC,patient_Recall@5_difference,51,DeepImmuno-CNN,0.0 +rcc_vaccine_benchmark,BigMHC,patient_Recall@10_difference,51,DeepImmuno-CNN,0.023809523809523725 +rcc_vaccine_benchmark,BigMHC,patient_Recall@20_difference,51,DeepImmuno-CNN,0.0 +rcc_vaccine_benchmark,BigMHC,record_score_spearman,128,PRIME,0.43052596790413744 +rcc_vaccine_benchmark,BigMHC,patient_Recall@1_difference,128,PRIME,-0.005691839025172354 +rcc_vaccine_benchmark,BigMHC,patient_Recall@5_difference,128,PRIME,-0.12574955908289237 +rcc_vaccine_benchmark,BigMHC,patient_Recall@10_difference,128,PRIME,-0.004489337822671136 +rcc_vaccine_benchmark,BigMHC,patient_Recall@20_difference,128,PRIME,0.0 +rcc_vaccine_benchmark,DeepHLApan,record_score_spearman,51,DeepImmuno-CNN,0.6475276218076436 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,51,DeepImmuno-CNN,-0.004761904761904759 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,51,DeepImmuno-CNN,1.1102230246251565e-16 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,51,DeepImmuno-CNN,0.04761904761904756 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,51,DeepImmuno-CNN,0.0 +rcc_vaccine_benchmark,DeepHLApan,record_score_spearman,128,PRIME,0.0359305767577579 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,128,PRIME,0.004409171075837742 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,128,PRIME,-0.12217412217412216 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,128,PRIME,0.006092672759339446 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,128,PRIME,0.0 +rcc_vaccine_benchmark,DeepImmuno-CNN,record_score_spearman,51,PRIME,0.2156361084619998 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@1_difference,51,PRIME,0.028571428571428567 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@5_difference,51,PRIME,-0.03571428571428581 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@10_difference,51,PRIME,-0.04761904761904756 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@20_difference,51,PRIME,0.0 diff --git a/results/analysis/stability/sensitivity_summary.csv b/results/analysis/stability/sensitivity_summary.csv new file mode 100644 index 0000000..4135855 --- /dev/null +++ b/results/analysis/stability/sensitivity_summary.csv @@ -0,0 +1,13 @@ +comparison,dataset,k,metric,predictors_in_support,strategy,value +BigMHC_vs_PRIME,improve_benchmark,5,probability_BigMHC_below_PRIME,,common_support,0.9185 +BigMHC_vs_PRIME,zhao_vaccine_benchmark,5,probability_BigMHC_below_PRIME,,common_support,0.0555 +BigMHC_vs_PRIME,rcc_vaccine_benchmark,5,probability_BigMHC_below_PRIME,,common_support,0.9985 +coverage_filtered_predictors,improve_benchmark,5,common_support_n,BigMHC|MHCflurry|PRIME,coverage_threshold_0.5,17475 +coverage_filtered_predictors,zhao_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|MHCflurry|PRIME,coverage_threshold_0.5,2310 +coverage_filtered_predictors,rcc_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|PRIME,coverage_threshold_0.5,128 +coverage_filtered_predictors,improve_benchmark,5,common_support_n,BigMHC|MHCflurry|PRIME,coverage_threshold_0.8,17475 +coverage_filtered_predictors,zhao_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|MHCflurry|PRIME,coverage_threshold_0.8,2310 +coverage_filtered_predictors,rcc_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|PRIME,coverage_threshold_0.8,128 +coverage_filtered_predictors,improve_benchmark,5,common_support_n,BigMHC|MHCflurry|PRIME,coverage_threshold_0.95,17475 +coverage_filtered_predictors,zhao_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|MHCflurry|PRIME,coverage_threshold_0.95,2310 +coverage_filtered_predictors,rcc_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|PRIME,coverage_threshold_0.95,128 diff --git a/results/manifest.json b/results/manifest.json index eb9ceaf..a03be78 100644 --- a/results/manifest.json +++ b/results/manifest.json @@ -156,9 +156,9 @@ "sha256": "1b2e9de3c7f7dfabc9c3241b7552db3cd8f7298e866a8f08ed2fe00a000b7635" }, { - "bytes": 487, + "bytes": 507, "path": "reports/manuscript_resource_audit.json", - "sha256": "90836e1d37e0eeaab46a0ab3781bc8811877c4c514570015d1546875fd3cdcd0" + "sha256": "f3ecb57f6e562c5737aa5755fe2a42a0f2cafb304aa18b9e45c7f7c6d4e39a10" }, { "bytes": 11059, @@ -216,14 +216,14 @@ "sha256": "537cb1397a68c114f4e49db1dd07be57fcab435bc2849d342f55399b650ac18e" }, { - "bytes": 28603, + "bytes": 32229, "path": "paper/manuscript_resource.md", - "sha256": "3f994070a10d1408aa2d985ced7947f2406f62eb41504c161166bb768d9839ac" + "sha256": "eab98d902505e17a8d312b1575977440ab4854d77ef91ef6a9aa844a4e5e4e54" }, { - "bytes": 4697, + "bytes": 5021, "path": "paper/references.bib", - "sha256": "5d883804d7d2be339135d8b3aa9efe2f4798ecadf4405a25b6318848c210216a" + "sha256": "272e420b33156882a5c53fed86b7c1eb22bdb7d29da916967fbc68f0f60266c1" }, { "bytes": 5320, @@ -236,9 +236,9 @@ "sha256": "3bc706c7b062876fcdec8c1fb12c788baacb2302ed4eb4beef67961ded4ac792" }, { - "bytes": 7353, + "bytes": 8519, "path": "FINAL_REPORT.md", - "sha256": "c2ffc0f9b83d7407a5e05ad36016e3cab9ccdf9ca7ab1dbc2f570f91bf7e4963" + "sha256": "16b3f44385a6e0fa7839bb3c3246f8695871a526989875665283aa08d55127f2" }, { "bytes": 1239894, @@ -369,6 +369,181 @@ "bytes": 288, "path": "reports/extension_clean_reproduction.json", "sha256": "f60acbb3d208c6971bf129be0c5d7be39dd2784147918005ff794b2ead558fa7" + }, + { + "bytes": 49526, + "path": "data/processed/rcc_vaccine_benchmark.csv", + "sha256": "1697b5808a05836995fbc8e8167d3516187f4184367fa7e1c2564b4701f6b3f8" + }, + { + "bytes": 49526, + "path": "data/processed/rcc_vaccine_benchmark_full.csv", + "sha256": "1697b5808a05836995fbc8e8167d3516187f4184367fa7e1c2564b4701f6b3f8" + }, + { + "bytes": 288, + "path": "data/rcc_vaccine_summary.json", + "sha256": "ff829adc508c37ca50fe06d5ddf7505c51373f837dcd8f06770bebad85eb0977" + }, + { + "bytes": 2861, + "path": "research/extension_protocol_rcc_v1.json", + "sha256": "fa3e89478b91e03500f1ad65327fe1a803624ea5ae88e782a1b466932fd2b399" + }, + { + "bytes": 12340, + "path": "research/training_overlap_audit_rcc.csv", + "sha256": "4381cd1b1a2b3061a8e26bb4d7deec8ff4606eb115e6c2ade4d3787051cbdb53" + }, + { + "bytes": 1070, + "path": "research/training_overlap_summary_rcc.json", + "sha256": "f8315e6c6115fbbd01fdb276d9bdbb68f7d9911c6da89a77c6e037ac7da42721" + }, + { + "bytes": 13826, + "path": "results/raw_predictions/rcc/bigmhc-v1.0.csv", + "sha256": "1b16e7bf545032a9763a9c899190775dddf24ea19378d8763899663a460f8dd5" + }, + { + "bytes": 12737, + "path": "results/raw_predictions/rcc/prime-2.0.csv", + "sha256": "68ed9f4f7a928964ccbab018d315e0e3ff9315d1b5c5a80665c30dee9a861140" + }, + { + "bytes": 13004, + "path": "results/raw_predictions/rcc/deepimmuno-cnn.csv", + "sha256": "be8fdb1f4efff6ef3159971a46f58513e8cc90d2d2f5737494aa14871700ad1e" + }, + { + "bytes": 15155, + "path": "results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "sha256": "4d084a12522852fbdfe3bcbe9b64a8477f496b2073e6f8fc95943bfd18b3e4d4" + }, + { + "bytes": 5035, + "path": "reports/rcc_predictor_run.json", + "sha256": "c5efc343344eff9f5c962f8d7834c7f3c0c686bcc669eaaf87617b44f0da3b00" + }, + { + "bytes": 83351, + "path": "results/analysis/rcc/metrics.json", + "sha256": "25ebbf73dd025d75b9d94f646022dfa4eaa794a3f80f47d81478cbe93df1fe60" + }, + { + "bytes": 266, + "path": "results/analysis/rcc/missingness.csv", + "sha256": "75b876d69948ea61012702b39a0382bed0a4b41ed40890c4985a0827a14a1d33" + }, + { + "bytes": 11452, + "path": "results/analysis/rcc/paired_differences.csv", + "sha256": "97e77b6a54cd871b34e9531cda9ef8e321fd6f2481387379d04c40475c365be8" + }, + { + "bytes": 1402, + "path": "results/analysis/stability/analysis_metadata.json", + "sha256": "a09048edd10e12ae98583d304f2f21c960f222a42ed6ca46f4089c43c6b46e3e" + }, + { + "bytes": 4538, + "path": "results/analysis/stability/dataset_predictor_metric_matrix.csv", + "sha256": "f8828927ff5b5aac296ad562145d95bc2dc50768478c512771f8044a80b6c0bb" + }, + { + "bytes": 8609, + "path": "results/analysis/stability/rank_stability.csv", + "sha256": "5c8b4b111f938788827184399f3fd063033fde55c33f8d22c1f9683bdbba2101" + }, + { + "bytes": 760, + "path": "results/analysis/stability/model_selection_first_probability.csv", + "sha256": "0ab7bae116a8a227aa079b35bafc1a17bc691cd121547c75f2975d176f028c14" + }, + { + "bytes": 1463, + "path": "results/analysis/stability/sensitivity_summary.csv", + "sha256": "96ec210d87df0fae602a35da0e2a9cdcd7554ad7aa8de9059d685525de20a8a4" + }, + { + "bytes": 1913, + "path": "results/analysis/stability/leave_one_domain_out.csv", + "sha256": "d35e7c8f98211daa299761df8a9cafdbd68528fae5d50252c55a25017a7cac7b" + }, + { + "bytes": 375, + "path": "results/analysis/stability/endpoint_domain_metadata.csv", + "sha256": "bb381081a5abc3cf45265d73aa812675fd2d7eac8319560eee4e7f918d1f0170" + }, + { + "bytes": 3655, + "path": "results/analysis/stability/endpoint_domain_auroc.svg", + "sha256": "8f9875a1ff7b0e75be13cd4c31e7623a1d4d14bd45481a8b2dad85e2abffb9ab" + }, + { + "bytes": 91312, + "path": "results/analysis/stability/endpoint_domain_auroc.png", + "sha256": "327fcd4d215d48f18832cf77959e125fc2d2f64490a80bd86cf3c9dd316c956b" + }, + { + "bytes": 348, + "path": "contracts/dataset-card.example.json", + "sha256": "4c3abe7cd9c09d8560df1b7c2530461fec81645442917c50969b0bed379c9c67" + }, + { + "bytes": 360, + "path": "contracts/predictor-card.example.json", + "sha256": "f2f0c6e1490f622e45b4c1b2e04076f0836632cf6aff08c92c70044be045a7cd" + }, + { + "bytes": 575, + "path": "contracts/prediction-artifact.schema.json", + "sha256": "ed38da54c4fc933ca616d3d88c57e894b97c6dc079962e542f6d5eae5ba32cbe" + }, + { + "bytes": 153, + "path": "contracts/synthetic/benchmark.csv", + "sha256": "06696235cd66940fe2bc88b15a1fb37cc5d085331ff652a1e393fde53b9cb310" + }, + { + "bytes": 283, + "path": "contracts/synthetic/predictions.csv", + "sha256": "ed054ea5f4da890ad676049d297eddc0243375de392a59594655ea24c56db858" + }, + { + "bytes": 321, + "path": "predictors/mhcmatch/attempts/sweep_receipt.json", + "sha256": "98b8977360a1a9f1fe031edf02692a7bbdf4c966c87cd7060d9528843126d575" + }, + { + "bytes": 321, + "path": "predictors/mhcnuggets/attempts/sweep_receipt.json", + "sha256": "f330557e0a8b5c092b4ab06bc18062233ae9c832021ff7419fde61b5d95d34ff" + }, + { + "bytes": 322, + "path": "predictors/neofox/attempts/sweep_receipt.json", + "sha256": "820962cf5c57c19542e89f45a34135ef414c186a9b5cfcf58894b3a6c0e64073" + }, + { + "bytes": 325, + "path": "predictors/neoguider/attempts/sweep_receipt.json", + "sha256": "5fee464b36230c75e18e02655dfa7dba45cca11a8c7062baf33201fb86cef123" + }, + { + "bytes": 344, + "path": "predictors/pvactools/attempts/sweep_receipt.json", + "sha256": "fdd6e350bdd290a5bd6d92a0d4c88e12203e185d96910f7c5cf0d2764361b18d" + }, + { + "bytes": 315, + "path": "predictors/seq2neo/attempts/sweep_receipt.json", + "sha256": "9e65865499af2e52047fa59bbac63a68484a5ef4cd5e65b3b6dbe68671bf26c7" + }, + { + "bytes": 312, + "path": "predictors/vaxrank/attempts/sweep_receipt.json", + "sha256": "8cb0d6206ce785f351b09acff12d0676462744653a48856608bc183a773d9365" } ], "benchmark": { @@ -425,7 +600,7 @@ "primary_metric": "patient-macro NDCG@5", "records": 2315 }, - "generated_date": "2026-08-20", + "generated_date": "2026-08-21", "predictor_revisions": { "BigMHC": "v1.0", "DeepHLApan": "1.1.1@ac1f4beb", @@ -434,7 +609,7 @@ "MHCflurry": "2.2.1", "PRIME": "2.0" }, - "project_commit": "f02559b72773293c175c7bde418206b4e39b37a9", + "project_commit": "db00e769f8d6831221264e6a187309cfeadbd91a", "project_worktree_clean": false, "schema_version": "1.0" } diff --git a/results/raw_predictions/rcc/bigmhc-v1.0.csv b/results/raw_predictions/rcc/bigmhc-v1.0.csv new file mode 100644 index 0000000..bbfbfa0 --- /dev/null +++ b/results/raw_predictions/rcc/bigmhc-v1.0.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,bigmhc_im,bigmhc_el +rcc-vaccine-8c2e1ec6d8f675cf,BigMHC,v1.0,immunogenicity,0.8492275,higher,predicted,0.8492275,0.998158 +rcc-vaccine-6a6ffb8b81ec9208,BigMHC,v1.0,immunogenicity,0.58425915,higher,predicted,0.58425915,0.16537319 +rcc-vaccine-4adeedbe0e35cf09,BigMHC,v1.0,immunogenicity,,higher,unsupported_hla,, +rcc-vaccine-e06799bdf8de8af5,BigMHC,v1.0,immunogenicity,0.8492275,higher,predicted,0.8492275,0.998158 +rcc-vaccine-969998ab9d8b4dc1,BigMHC,v1.0,immunogenicity,0.00042694155,higher,predicted,0.00042694155,0.0001798583 +rcc-vaccine-a5952e9e71bd27bd,BigMHC,v1.0,immunogenicity,0.19695829,higher,predicted,0.19695829,0.15335162 +rcc-vaccine-6fa3e9ad450e030e,BigMHC,v1.0,immunogenicity,0.8492275,higher,predicted,0.8492275,0.998158 +rcc-vaccine-8a9041cc55b08e38,BigMHC,v1.0,immunogenicity,0.0199301,higher,predicted,0.0199301,0.026195873 +rcc-vaccine-e11c81feedc5751e,BigMHC,v1.0,immunogenicity,0.03303969,higher,predicted,0.03303969,0.003229972 +rcc-vaccine-fefffdcccca8649c,BigMHC,v1.0,immunogenicity,0.2625458,higher,predicted,0.2625458,0.5432058 +rcc-vaccine-940b62df9c3078c4,BigMHC,v1.0,immunogenicity,0.051157277,higher,predicted,0.051157277,0.3431107 +rcc-vaccine-43742d14e082c2e6,BigMHC,v1.0,immunogenicity,0.00051022216,higher,predicted,0.00051022216,1.4941308e-05 +rcc-vaccine-769c6b305827c570,BigMHC,v1.0,immunogenicity,0.17434175,higher,predicted,0.17434175,0.014034523 +rcc-vaccine-e74f183589095f61,BigMHC,v1.0,immunogenicity,0.14538826,higher,predicted,0.14538826,0.0007323162 +rcc-vaccine-15ab70cb934aad8f,BigMHC,v1.0,immunogenicity,0.1274003,higher,predicted,0.1274003,0.0017386841 +rcc-vaccine-c8a8ce5ab15f9d4b,BigMHC,v1.0,immunogenicity,0.07331483,higher,predicted,0.07331483,0.0027492247 +rcc-vaccine-f05602883865f4bf,BigMHC,v1.0,immunogenicity,0.094515726,higher,predicted,0.094515726,0.41674283 +rcc-vaccine-b92105a12fb6f028,BigMHC,v1.0,immunogenicity,0.1498395,higher,predicted,0.1498395,0.5488981 +rcc-vaccine-a004075aed453bcf,BigMHC,v1.0,immunogenicity,0.19785257,higher,predicted,0.19785257,0.007729145 +rcc-vaccine-70eb710f90e2973b,BigMHC,v1.0,immunogenicity,0.037451383,higher,predicted,0.037451383,0.05899397 +rcc-vaccine-c81528933be4a147,BigMHC,v1.0,immunogenicity,0.032642193,higher,predicted,0.032642193,0.7174736 +rcc-vaccine-2a2f0b8f959aa3f6,BigMHC,v1.0,immunogenicity,0.22346842,higher,predicted,0.22346842,0.07276444 +rcc-vaccine-74e8a99b7ef86c61,BigMHC,v1.0,immunogenicity,0.6325035,higher,predicted,0.6325035,0.9122117 +rcc-vaccine-88bea2527dd72a1d,BigMHC,v1.0,immunogenicity,0.11432793,higher,predicted,0.11432793,0.012796485 +rcc-vaccine-ff6bbb8d69468a47,BigMHC,v1.0,immunogenicity,0.07834623,higher,predicted,0.07834623,0.12306678 +rcc-vaccine-8ea6c4f21d2c81ee,BigMHC,v1.0,immunogenicity,0.0004803969,higher,predicted,0.0004803969,0.00025231813 +rcc-vaccine-b2f6110d808ac0da,BigMHC,v1.0,immunogenicity,0.12104134,higher,predicted,0.12104134,0.04732251 +rcc-vaccine-f43e93564087f160,BigMHC,v1.0,immunogenicity,0.112379804,higher,predicted,0.112379804,0.6370787 +rcc-vaccine-fc1276f53c3f45e1,BigMHC,v1.0,immunogenicity,0.013032401,higher,predicted,0.013032401,0.00011015074 +rcc-vaccine-3e6b1c1d8de0ce71,BigMHC,v1.0,immunogenicity,0.10996994,higher,predicted,0.10996994,0.0041871676 +rcc-vaccine-4255bdded9e984bd,BigMHC,v1.0,immunogenicity,0.20445038,higher,predicted,0.20445038,0.3118517 +rcc-vaccine-31c3a5a1c962db43,BigMHC,v1.0,immunogenicity,0.33050114,higher,predicted,0.33050114,0.98251694 +rcc-vaccine-d3d4046d155047b6,BigMHC,v1.0,immunogenicity,0.4889801,higher,predicted,0.4889801,0.6427992 +rcc-vaccine-5e4096b57e93c92e,BigMHC,v1.0,immunogenicity,0.013662019,higher,predicted,0.013662019,0.048441563 +rcc-vaccine-e8d6aeb51705642a,BigMHC,v1.0,immunogenicity,0.035264768,higher,predicted,0.035264768,0.3544042 +rcc-vaccine-26dc24e186221fe2,BigMHC,v1.0,immunogenicity,0.120982565,higher,predicted,0.120982565,0.35466275 +rcc-vaccine-364822c022710958,BigMHC,v1.0,immunogenicity,0.6495637,higher,predicted,0.6495637,0.4203276 +rcc-vaccine-9c477d13a3359f5e,BigMHC,v1.0,immunogenicity,0.2634864,higher,predicted,0.2634864,0.0073506013 +rcc-vaccine-487a8e43d12c17ac,BigMHC,v1.0,immunogenicity,0.64040625,higher,predicted,0.64040625,0.69878644 +rcc-vaccine-539b50b223ea0ee1,BigMHC,v1.0,immunogenicity,0.15084496,higher,predicted,0.15084496,0.95499474 +rcc-vaccine-618d6aa7d8232657,BigMHC,v1.0,immunogenicity,0.1541317,higher,predicted,0.1541317,0.0509347 +rcc-vaccine-71f5d2ab19a389e8,BigMHC,v1.0,immunogenicity,0.012482004,higher,predicted,0.012482004,0.16006315 +rcc-vaccine-409a0661a319bbb1,BigMHC,v1.0,immunogenicity,0.2634864,higher,predicted,0.2634864,0.0073506013 +rcc-vaccine-9f431d040990f0f7,BigMHC,v1.0,immunogenicity,0.018202517,higher,predicted,0.018202517,0.63248533 +rcc-vaccine-7c1009a0bb0524b3,BigMHC,v1.0,immunogenicity,0.02231897,higher,predicted,0.02231897,0.0074036336 +rcc-vaccine-42e869aba90dc4bd,BigMHC,v1.0,immunogenicity,0.00010520963,higher,predicted,0.00010520963,8.958782e-05 +rcc-vaccine-a1a88d1c11b4fcdd,BigMHC,v1.0,immunogenicity,0.04779095,higher,predicted,0.04779095,0.0033836025 +rcc-vaccine-d42b9fb3a44366ac,BigMHC,v1.0,immunogenicity,0.15084496,higher,predicted,0.15084496,0.95499474 +rcc-vaccine-ce806d8b1a4132b2,BigMHC,v1.0,immunogenicity,0.1408995,higher,predicted,0.1408995,0.8896032 +rcc-vaccine-ea51c18cc4e16995,BigMHC,v1.0,immunogenicity,0.1541317,higher,predicted,0.1541317,0.0509347 +rcc-vaccine-9bce007ba3a4aff0,BigMHC,v1.0,immunogenicity,0.010021402,higher,predicted,0.010021402,0.0002278478 +rcc-vaccine-aa3019acbdddb7dc,BigMHC,v1.0,immunogenicity,0.020995839,higher,predicted,0.020995839,0.00015733231 +rcc-vaccine-1bcdaed331509a08,BigMHC,v1.0,immunogenicity,0.1408995,higher,predicted,0.1408995,0.8896032 +rcc-vaccine-4a1853322eea7a14,BigMHC,v1.0,immunogenicity,0.008962676,higher,predicted,0.008962676,0.00016499302 +rcc-vaccine-cde49959d75e6ec2,BigMHC,v1.0,immunogenicity,0.034591164,higher,predicted,0.034591164,0.05231158 +rcc-vaccine-7f8e1f8e02332663,BigMHC,v1.0,immunogenicity,0.028659904,higher,predicted,0.028659904,0.8122644 +rcc-vaccine-b8aa4d97d214f765,BigMHC,v1.0,immunogenicity,0.04871258,higher,predicted,0.04871258,0.0059868065 +rcc-vaccine-5993797f7a6c7057,BigMHC,v1.0,immunogenicity,0.1504514,higher,predicted,0.1504514,0.078459814 +rcc-vaccine-eacb2f310488ca9d,BigMHC,v1.0,immunogenicity,0.25954965,higher,predicted,0.25954965,0.43180725 +rcc-vaccine-70f05fcfa038224f,BigMHC,v1.0,immunogenicity,0.0045895926,higher,predicted,0.0045895926,0.06761794 +rcc-vaccine-4d15c4e62bb19311,BigMHC,v1.0,immunogenicity,0.0026878978,higher,predicted,0.0026878978,0.0007637069 +rcc-vaccine-a6b2180415ff1b56,BigMHC,v1.0,immunogenicity,0.02717142,higher,predicted,0.02717142,0.17152806 +rcc-vaccine-e52f3609d694b4ba,BigMHC,v1.0,immunogenicity,0.15067343,higher,predicted,0.15067343,0.01708421 +rcc-vaccine-763f29ff7ac4c65d,BigMHC,v1.0,immunogenicity,0.10464774,higher,predicted,0.10464774,0.017464515 +rcc-vaccine-c331f65899396b9b,BigMHC,v1.0,immunogenicity,0.05073751,higher,predicted,0.05073751,0.16568084 +rcc-vaccine-09c9b699d977f421,BigMHC,v1.0,immunogenicity,0.056319688,higher,predicted,0.056319688,0.6093203 +rcc-vaccine-e944d5a782a7d074,BigMHC,v1.0,immunogenicity,0.40012008,higher,predicted,0.40012008,0.039899163 +rcc-vaccine-75f86cf28dca60e2,BigMHC,v1.0,immunogenicity,0.076639615,higher,predicted,0.076639615,0.118811674 +rcc-vaccine-5b2ef450eaf2248a,BigMHC,v1.0,immunogenicity,0.21340606,higher,predicted,0.21340606,0.88489115 +rcc-vaccine-c50e7f0e1b3b69a5,BigMHC,v1.0,immunogenicity,0.22124277,higher,predicted,0.22124277,0.5948638 +rcc-vaccine-ae70f68909b113e8,BigMHC,v1.0,immunogenicity,0.10368972,higher,predicted,0.10368972,0.05833147 +rcc-vaccine-efdf21635c142ab8,BigMHC,v1.0,immunogenicity,0.012091301,higher,predicted,0.012091301,0.013448933 +rcc-vaccine-817b1ac6328997a7,BigMHC,v1.0,immunogenicity,0.0039570313,higher,predicted,0.0039570313,0.03172536 +rcc-vaccine-feb66d3c66c8a4de,BigMHC,v1.0,immunogenicity,0.14428535,higher,predicted,0.14428535,0.55323803 +rcc-vaccine-7946e9a10fe114cf,BigMHC,v1.0,immunogenicity,0.009083572,higher,predicted,0.009083572,0.007149222 +rcc-vaccine-f3e32e857dd09d71,BigMHC,v1.0,immunogenicity,0.098906696,higher,predicted,0.098906696,0.42554325 +rcc-vaccine-240bf5b232815c6a,BigMHC,v1.0,immunogenicity,0.35946846,higher,predicted,0.35946846,0.5475027 +rcc-vaccine-40c01a17a3ed4a2e,BigMHC,v1.0,immunogenicity,0.04898708,higher,predicted,0.04898708,0.11417298 +rcc-vaccine-7b823cc728ea1450,BigMHC,v1.0,immunogenicity,0.08569368,higher,predicted,0.08569368,0.9577692 +rcc-vaccine-0c7eb5e3295f0ad2,BigMHC,v1.0,immunogenicity,0.012339222,higher,predicted,0.012339222,0.09743548 +rcc-vaccine-88aaf2311ce63196,BigMHC,v1.0,immunogenicity,0.14953732,higher,predicted,0.14953732,0.11653518 +rcc-vaccine-bcae618cfad77c1f,BigMHC,v1.0,immunogenicity,0.019738207,higher,predicted,0.019738207,0.086946145 +rcc-vaccine-11f61d47947266cc,BigMHC,v1.0,immunogenicity,0.046728577,higher,predicted,0.046728577,0.43281174 +rcc-vaccine-f4a4db2362e7a54e,BigMHC,v1.0,immunogenicity,0.58805186,higher,predicted,0.58805186,0.7872415 +rcc-vaccine-1eceb5b6a969d721,BigMHC,v1.0,immunogenicity,0.64963526,higher,predicted,0.64963526,0.73666704 +rcc-vaccine-55106745d24cfbf4,BigMHC,v1.0,immunogenicity,0.15317993,higher,predicted,0.15317993,0.2744644 +rcc-vaccine-a808947689fbe42c,BigMHC,v1.0,immunogenicity,0.037526943,higher,predicted,0.037526943,0.4084019 +rcc-vaccine-590f5ea08f24e2cf,BigMHC,v1.0,immunogenicity,0.11108919,higher,predicted,0.11108919,0.03396273 +rcc-vaccine-b984dc3be5c1fa17,BigMHC,v1.0,immunogenicity,0.58805186,higher,predicted,0.58805186,0.7872415 +rcc-vaccine-6ff6e758ac279425,BigMHC,v1.0,immunogenicity,0.20403412,higher,predicted,0.20403412,0.030309463 +rcc-vaccine-aa0d4e0503dacba4,BigMHC,v1.0,immunogenicity,0.096679814,higher,predicted,0.096679814,0.043357793 +rcc-vaccine-4d44829f31cf5558,BigMHC,v1.0,immunogenicity,0.09515708,higher,predicted,0.09515708,0.33479306 +rcc-vaccine-4f7d9f9f22d48440,BigMHC,v1.0,immunogenicity,0.31902334,higher,predicted,0.31902334,0.22489502 +rcc-vaccine-60d1ba1b8ae350fd,BigMHC,v1.0,immunogenicity,0.4844474,higher,predicted,0.4844474,0.8657421 +rcc-vaccine-7ec85836cb09ee30,BigMHC,v1.0,immunogenicity,0.008745876,higher,predicted,0.008745876,0.08516661 +rcc-vaccine-4b7fdcc54f65b98b,BigMHC,v1.0,immunogenicity,0.4844474,higher,predicted,0.4844474,0.8657421 +rcc-vaccine-d02de725ecc685a5,BigMHC,v1.0,immunogenicity,0.3566296,higher,predicted,0.3566296,0.11564131 +rcc-vaccine-e1d984ebf245728f,BigMHC,v1.0,immunogenicity,0.09470768,higher,predicted,0.09470768,0.08543001 +rcc-vaccine-a9a3c2bc98a74746,BigMHC,v1.0,immunogenicity,0.03360454,higher,predicted,0.03360454,0.64224684 +rcc-vaccine-da610226e338a8de,BigMHC,v1.0,immunogenicity,0.48562998,higher,predicted,0.48562998,0.72100705 +rcc-vaccine-ca0a8a8bb4e2d094,BigMHC,v1.0,immunogenicity,0.07511683,higher,predicted,0.07511683,0.0056161615 +rcc-vaccine-c45585f253341070,BigMHC,v1.0,immunogenicity,0.38412148,higher,predicted,0.38412148,0.8916814 +rcc-vaccine-b2761351ccb757b7,BigMHC,v1.0,immunogenicity,0.03360454,higher,predicted,0.03360454,0.64224684 +rcc-vaccine-b1ad936ccc2f249a,BigMHC,v1.0,immunogenicity,0.38412148,higher,predicted,0.38412148,0.8916814 +rcc-vaccine-02261377bb800592,BigMHC,v1.0,immunogenicity,0.009326143,higher,predicted,0.009326143,0.20740989 +rcc-vaccine-834aff6d2a7dc363,BigMHC,v1.0,immunogenicity,0.06686703,higher,predicted,0.06686703,0.07113467 +rcc-vaccine-0036b8c45a4cd20d,BigMHC,v1.0,immunogenicity,0.03360454,higher,predicted,0.03360454,0.64224684 +rcc-vaccine-08447808dd0300dd,BigMHC,v1.0,immunogenicity,0.38412148,higher,predicted,0.38412148,0.8916814 +rcc-vaccine-34c3e7e7a2dcd8b6,BigMHC,v1.0,immunogenicity,0.009326143,higher,predicted,0.009326143,0.20740989 +rcc-vaccine-1ae968c5ae01f49b,BigMHC,v1.0,immunogenicity,0.06686703,higher,predicted,0.06686703,0.07113467 +rcc-vaccine-29f7c92d094c1ced,BigMHC,v1.0,immunogenicity,0.040688813,higher,predicted,0.040688813,0.55616236 +rcc-vaccine-0aaef5904890371c,BigMHC,v1.0,immunogenicity,0.06686703,higher,predicted,0.06686703,0.07113467 +rcc-vaccine-27b47d43dab2b353,BigMHC,v1.0,immunogenicity,0.69446176,higher,predicted,0.69446176,0.30978775 +rcc-vaccine-46b486b9d199e93a,BigMHC,v1.0,immunogenicity,0.13332154,higher,predicted,0.13332154,0.026609246 +rcc-vaccine-b8a9c44bcf53568f,BigMHC,v1.0,immunogenicity,0.30859342,higher,predicted,0.30859342,0.4905243 +rcc-vaccine-7425139a6230adc0,BigMHC,v1.0,immunogenicity,0.35320142,higher,predicted,0.35320142,0.6920496 +rcc-vaccine-e39c49c350f42009,BigMHC,v1.0,immunogenicity,0.14485602,higher,predicted,0.14485602,0.90685815 +rcc-vaccine-73717d4619b66faf,BigMHC,v1.0,immunogenicity,0.3526861,higher,predicted,0.3526861,0.08524216 +rcc-vaccine-f130cbb5ea1ca0fa,BigMHC,v1.0,immunogenicity,0.003947547,higher,predicted,0.003947547,0.0018933918 +rcc-vaccine-3f16099c911d2791,BigMHC,v1.0,immunogenicity,0.35320142,higher,predicted,0.35320142,0.6920496 +rcc-vaccine-07bc8054e1edd65a,BigMHC,v1.0,immunogenicity,0.41273668,higher,predicted,0.41273668,0.94675225 +rcc-vaccine-99bd603a759d6b22,BigMHC,v1.0,immunogenicity,0.50416905,higher,predicted,0.50416905,0.8801918 +rcc-vaccine-ad726d8ef91bd0e6,BigMHC,v1.0,immunogenicity,0.243946,higher,predicted,0.243946,0.0207863 +rcc-vaccine-972e1f28fe4cb96a,BigMHC,v1.0,immunogenicity,0.0084418915,higher,predicted,0.0084418915,0.0074836127 +rcc-vaccine-e0d628668c1eccf2,BigMHC,v1.0,immunogenicity,0.007305874,higher,predicted,0.007305874,5.319204e-05 +rcc-vaccine-3d165d772065bb3a,BigMHC,v1.0,immunogenicity,0.015850548,higher,predicted,0.015850548,0.61480916 +rcc-vaccine-9ef49682993d2ea1,BigMHC,v1.0,immunogenicity,0.024873625,higher,predicted,0.024873625,0.0945738 +rcc-vaccine-6c01a27a6a575007,BigMHC,v1.0,immunogenicity,0.21619721,higher,predicted,0.21619721,0.08226664 +rcc-vaccine-12b152d35907e280,BigMHC,v1.0,immunogenicity,0.09390874,higher,predicted,0.09390874,0.6517529 diff --git a/results/raw_predictions/rcc/deephlapan-1.1.1.csv b/results/raw_predictions/rcc/deephlapan-1.1.1.csv new file mode 100644 index 0000000..862d834 --- /dev/null +++ b/results/raw_predictions/rcc/deephlapan-1.1.1.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,binding_score +rcc-vaccine-8c2e1ec6d8f675cf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.956750869751,higher,predicted,0.999957203865 +rcc-vaccine-6a6ffb8b81ec9208,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0342342741787,higher,predicted,0.995593428612 +rcc-vaccine-4adeedbe0e35cf09,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,,higher,unsupported_hla, +rcc-vaccine-e06799bdf8de8af5,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.956750869751,higher,predicted,0.999957203865 +rcc-vaccine-969998ab9d8b4dc1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.10936331749,higher,predicted,0.969243168831 +rcc-vaccine-a5952e9e71bd27bd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.672411322594,higher,predicted,0.970010876656 +rcc-vaccine-6fa3e9ad450e030e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.956750869751,higher,predicted,0.999957203865 +rcc-vaccine-8a9041cc55b08e38,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.415020555258,higher,predicted,0.823912143707 +rcc-vaccine-e11c81feedc5751e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.271832853556,higher,predicted,0.853784382343 +rcc-vaccine-fefffdcccca8649c,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.663122951984,higher,predicted,0.990222096443 +rcc-vaccine-940b62df9c3078c4,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.288684308529,higher,predicted,0.945045113564 +rcc-vaccine-43742d14e082c2e6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0819523632526,higher,predicted,0.478505432606 +rcc-vaccine-769c6b305827c570,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.949861049652,higher,predicted,0.997111976147 +rcc-vaccine-e74f183589095f61,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.44372677803,higher,predicted,0.493312120438 +rcc-vaccine-15ab70cb934aad8f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.708318173885,higher,predicted,0.989219069481 +rcc-vaccine-c8a8ce5ab15f9d4b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.773888707161,higher,predicted,0.0873938798904 +rcc-vaccine-f05602883865f4bf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.954157352448,higher,predicted,0.991816818714 +rcc-vaccine-b92105a12fb6f028,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.929277539253,higher,predicted,0.99683535099 +rcc-vaccine-a004075aed453bcf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.773697257042,higher,predicted,0.662531495094 +rcc-vaccine-70eb710f90e2973b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.638193964958,higher,predicted,0.979091465473 +rcc-vaccine-c81528933be4a147,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0489731431007,higher,predicted,0.960151314735 +rcc-vaccine-2a2f0b8f959aa3f6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.54920065403,higher,predicted,0.952247798443 +rcc-vaccine-74e8a99b7ef86c61,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.364848554134,higher,predicted,0.999667048454 +rcc-vaccine-88bea2527dd72a1d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.996774017811,higher,predicted,0.935574650764 +rcc-vaccine-ff6bbb8d69468a47,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.991322398186,higher,predicted,0.856530070305 +rcc-vaccine-8ea6c4f21d2c81ee,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.904866814613,higher,predicted,0.830239772797 +rcc-vaccine-b2f6110d808ac0da,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.900792896748,higher,predicted,0.475547015667 +rcc-vaccine-f43e93564087f160,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.981161594391,higher,predicted,0.982510209084 +rcc-vaccine-fc1276f53c3f45e1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.781159996986,higher,predicted,0.835298418999 +rcc-vaccine-3e6b1c1d8de0ce71,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.991661071777,higher,predicted,0.868157505989 +rcc-vaccine-4255bdded9e984bd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.797833561897,higher,predicted,0.99191236496 +rcc-vaccine-31c3a5a1c962db43,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.282823115587,higher,predicted,0.99037283659 +rcc-vaccine-d3d4046d155047b6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.980403542519,higher,predicted,0.202609583735 +rcc-vaccine-5e4096b57e93c92e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0258417315781,higher,predicted,0.151212573051 +rcc-vaccine-e8d6aeb51705642a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.995793521404,higher,predicted,0.207422688603 +rcc-vaccine-26dc24e186221fe2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.730694293976,higher,predicted,0.968062222004 +rcc-vaccine-364822c022710958,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.13982026279,higher,predicted,0.999332427979 +rcc-vaccine-9c477d13a3359f5e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.691724777222,higher,predicted,0.993195176125 +rcc-vaccine-487a8e43d12c17ac,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.668253481388,higher,predicted,0.998432040215 +rcc-vaccine-539b50b223ea0ee1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.418166160583,higher,predicted,0.998265862465 +rcc-vaccine-618d6aa7d8232657,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.731171905994,higher,predicted,0.997755050659 +rcc-vaccine-71f5d2ab19a389e8,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0762045830488,higher,predicted,0.461806923151 +rcc-vaccine-409a0661a319bbb1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.691724777222,higher,predicted,0.993195176125 +rcc-vaccine-9f431d040990f0f7,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00906158518046,higher,predicted,0.991863429546 +rcc-vaccine-7c1009a0bb0524b3,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.731387615204,higher,predicted,0.949160933495 +rcc-vaccine-42e869aba90dc4bd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.219947695732,higher,predicted,0.773827433586 +rcc-vaccine-a1a88d1c11b4fcdd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.757146000862,higher,predicted,0.971038460732 +rcc-vaccine-d42b9fb3a44366ac,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.418166160583,higher,predicted,0.998265862465 +rcc-vaccine-ce806d8b1a4132b2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.172158882022,higher,predicted,0.996777057648 +rcc-vaccine-ea51c18cc4e16995,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.731171905994,higher,predicted,0.997755050659 +rcc-vaccine-9bce007ba3a4aff0,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.460265338421,higher,predicted,0.826100230217 +rcc-vaccine-aa3019acbdddb7dc,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.657150745392,higher,predicted,0.952590823174 +rcc-vaccine-1bcdaed331509a08,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.172158882022,higher,predicted,0.996777057648 +rcc-vaccine-4a1853322eea7a14,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.461393415928,higher,predicted,0.986970543861 +rcc-vaccine-cde49959d75e6ec2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.75685107708,higher,predicted,0.979090809822 +rcc-vaccine-7f8e1f8e02332663,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0762795731425,higher,predicted,0.997413814068 +rcc-vaccine-b8aa4d97d214f765,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.990990817547,higher,predicted,0.999659538269 +rcc-vaccine-5993797f7a6c7057,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.442390143871,higher,predicted,0.993007063866 +rcc-vaccine-eacb2f310488ca9d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0443214699626,higher,predicted,0.999183535576 +rcc-vaccine-70f05fcfa038224f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.644423604012,higher,predicted,0.974320590496 +rcc-vaccine-4d15c4e62bb19311,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00129490566906,higher,predicted,0.908160805702 +rcc-vaccine-a6b2180415ff1b56,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.103126898408,higher,predicted,0.870270550251 +rcc-vaccine-e52f3609d694b4ba,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.249618858099,higher,predicted,0.946961700916 +rcc-vaccine-763f29ff7ac4c65d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0525496602058,higher,predicted,0.99926173687 +rcc-vaccine-c331f65899396b9b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.976519405842,higher,predicted,0.984708964825 +rcc-vaccine-09c9b699d977f421,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.816194355488,higher,predicted,0.991399765015 +rcc-vaccine-e944d5a782a7d074,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.533009707928,higher,predicted,0.962785363197 +rcc-vaccine-75f86cf28dca60e2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0963276177645,higher,predicted,0.998524844646 +rcc-vaccine-5b2ef450eaf2248a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.985190093517,higher,predicted,0.999938488007 +rcc-vaccine-c50e7f0e1b3b69a5,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0859711095691,higher,predicted,0.991966068745 +rcc-vaccine-ae70f68909b113e8,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00732829514891,higher,predicted,0.967723369598 +rcc-vaccine-efdf21635c142ab8,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00268213148229,higher,predicted,0.965277791023 +rcc-vaccine-817b1ac6328997a7,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.920276165009,higher,predicted,0.99510204792 +rcc-vaccine-feb66d3c66c8a4de,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.548630595207,higher,predicted,0.991096198559 +rcc-vaccine-7946e9a10fe114cf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.313266456127,higher,predicted,0.974083065987 +rcc-vaccine-f3e32e857dd09d71,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.02863131091,higher,predicted,0.999120116234 +rcc-vaccine-240bf5b232815c6a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.965816140175,higher,predicted,0.999469399452 +rcc-vaccine-40c01a17a3ed4a2e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.660215973854,higher,predicted,0.978103756905 +rcc-vaccine-7b823cc728ea1450,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.93551158905,higher,predicted,0.962181270123 +rcc-vaccine-0c7eb5e3295f0ad2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0372959189117,higher,predicted,0.999003231525 +rcc-vaccine-88aaf2311ce63196,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.959538161755,higher,predicted,0.999945759773 +rcc-vaccine-bcae618cfad77c1f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.255532085896,higher,predicted,0.933230400085 +rcc-vaccine-11f61d47947266cc,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.994814693928,higher,predicted,0.990790724754 +rcc-vaccine-f4a4db2362e7a54e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.576282143593,higher,predicted,0.998294651508 +rcc-vaccine-1eceb5b6a969d721,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.721192061901,higher,predicted,0.99988758564 +rcc-vaccine-55106745d24cfbf4,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.676800429821,higher,predicted,0.945376694202 +rcc-vaccine-a808947689fbe42c,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.800811767578,higher,predicted,0.99976503849 +rcc-vaccine-590f5ea08f24e2cf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.528627693653,higher,predicted,0.999486088753 +rcc-vaccine-b984dc3be5c1fa17,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.576282143593,higher,predicted,0.998294651508 +rcc-vaccine-6ff6e758ac279425,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.5315746665,higher,predicted,0.868035018444 +rcc-vaccine-aa0d4e0503dacba4,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.810282886028,higher,predicted,0.994153857231 +rcc-vaccine-4d44829f31cf5558,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.786226093769,higher,predicted,0.988708198071 +rcc-vaccine-4f7d9f9f22d48440,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.938049137592,higher,predicted,0.999858498573 +rcc-vaccine-60d1ba1b8ae350fd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.907981276512,higher,predicted,0.999875664711 +rcc-vaccine-7ec85836cb09ee30,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.567132294178,higher,predicted,0.990272343159 +rcc-vaccine-4b7fdcc54f65b98b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.907981276512,higher,predicted,0.999875664711 +rcc-vaccine-d02de725ecc685a5,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.716995000839,higher,predicted,0.998158454895 +rcc-vaccine-e1d984ebf245728f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.777408897877,higher,predicted,0.999284625053 +rcc-vaccine-a9a3c2bc98a74746,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0750500410795,higher,predicted,0.91932952404 +rcc-vaccine-da610226e338a8de,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.465691566467,higher,predicted,0.999276459217 +rcc-vaccine-ca0a8a8bb4e2d094,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.952795028687,higher,predicted,0.997379779816 +rcc-vaccine-c45585f253341070,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.494304269552,higher,predicted,0.996902823448 +rcc-vaccine-b2761351ccb757b7,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0750500410795,higher,predicted,0.91932952404 +rcc-vaccine-b1ad936ccc2f249a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.494304269552,higher,predicted,0.996902823448 +rcc-vaccine-02261377bb800592,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.129811197519,higher,predicted,0.999617755413 +rcc-vaccine-834aff6d2a7dc363,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.88037699461,higher,predicted,0.955722689629 +rcc-vaccine-0036b8c45a4cd20d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0750500410795,higher,predicted,0.91932952404 +rcc-vaccine-08447808dd0300dd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.494304269552,higher,predicted,0.996902823448 +rcc-vaccine-34c3e7e7a2dcd8b6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.129811197519,higher,predicted,0.999617755413 +rcc-vaccine-1ae968c5ae01f49b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.88037699461,higher,predicted,0.955722689629 +rcc-vaccine-29f7c92d094c1ced,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.165366202593,higher,predicted,0.998827159405 +rcc-vaccine-0aaef5904890371c,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.88037699461,higher,predicted,0.955722689629 +rcc-vaccine-27b47d43dab2b353,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.952636361122,higher,predicted,0.999851703644 +rcc-vaccine-46b486b9d199e93a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.814331412315,higher,predicted,0.790807783604 +rcc-vaccine-b8a9c44bcf53568f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.161298811436,higher,predicted,0.998297870159 +rcc-vaccine-7425139a6230adc0,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.889500260353,higher,predicted,0.977608501911 +rcc-vaccine-e39c49c350f42009,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.468123346567,higher,predicted,0.999036669731 +rcc-vaccine-73717d4619b66faf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.963266253471,higher,predicted,0.998651981354 +rcc-vaccine-f130cbb5ea1ca0fa,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.99343174696,higher,predicted,0.916508376598 +rcc-vaccine-3f16099c911d2791,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.889500260353,higher,predicted,0.977608501911 +rcc-vaccine-07bc8054e1edd65a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.314443677664,higher,predicted,0.999527335167 +rcc-vaccine-99bd603a759d6b22,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.950844585896,higher,predicted,0.970935940742 +rcc-vaccine-ad726d8ef91bd0e6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.969846844673,higher,predicted,0.992808938026 +rcc-vaccine-972e1f28fe4cb96a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.678239226341,higher,predicted,0.980034708977 +rcc-vaccine-e0d628668c1eccf2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.76940625906,higher,predicted,0.512591063976 +rcc-vaccine-3d165d772065bb3a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.982267737389,higher,predicted,0.99388551712 +rcc-vaccine-9ef49682993d2ea1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.435784041882,higher,predicted,0.955987811089 +rcc-vaccine-6c01a27a6a575007,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.963434100151,higher,predicted,0.847052097321 +rcc-vaccine-12b152d35907e280,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.205441281199,higher,predicted,0.999481081963 diff --git a/results/raw_predictions/rcc/deepimmuno-cnn.csv b/results/raw_predictions/rcc/deepimmuno-cnn.csv new file mode 100644 index 0000000..d6e548c --- /dev/null +++ b/results/raw_predictions/rcc/deepimmuno-cnn.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status +rcc-vaccine-8c2e1ec6d8f675cf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.936987876892,higher,predicted +rcc-vaccine-6a6ffb8b81ec9208,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-4adeedbe0e35cf09,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-e06799bdf8de8af5,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.936987876892,higher,predicted +rcc-vaccine-969998ab9d8b4dc1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-a5952e9e71bd27bd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.783586323261,higher,predicted +rcc-vaccine-6fa3e9ad450e030e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.936987876892,higher,predicted +rcc-vaccine-8a9041cc55b08e38,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.697320222855,higher,predicted +rcc-vaccine-e11c81feedc5751e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-fefffdcccca8649c,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.697254955769,higher,predicted +rcc-vaccine-940b62df9c3078c4,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-43742d14e082c2e6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-769c6b305827c570,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-e74f183589095f61,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-15ab70cb934aad8f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-c8a8ce5ab15f9d4b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.688341677189,higher,predicted +rcc-vaccine-f05602883865f4bf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-b92105a12fb6f028,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-a004075aed453bcf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-70eb710f90e2973b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-c81528933be4a147,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.374690830708,higher,predicted +rcc-vaccine-2a2f0b8f959aa3f6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-74e8a99b7ef86c61,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-88bea2527dd72a1d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.985417187214,higher,predicted +rcc-vaccine-ff6bbb8d69468a47,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-8ea6c4f21d2c81ee,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-b2f6110d808ac0da,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.838476061821,higher,predicted +rcc-vaccine-f43e93564087f160,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.8036480546,higher,predicted +rcc-vaccine-fc1276f53c3f45e1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.904640376568,higher,predicted +rcc-vaccine-3e6b1c1d8de0ce71,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.985883593559,higher,predicted +rcc-vaccine-4255bdded9e984bd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-31c3a5a1c962db43,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.761615693569,higher,predicted +rcc-vaccine-d3d4046d155047b6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-5e4096b57e93c92e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.408737450838,higher,predicted +rcc-vaccine-e8d6aeb51705642a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.988444507122,higher,predicted +rcc-vaccine-26dc24e186221fe2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.945877850056,higher,predicted +rcc-vaccine-364822c022710958,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-9c477d13a3359f5e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-487a8e43d12c17ac,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-539b50b223ea0ee1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-618d6aa7d8232657,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-71f5d2ab19a389e8,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.309658080339,higher,predicted +rcc-vaccine-409a0661a319bbb1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-9f431d040990f0f7,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.302717536688,higher,predicted +rcc-vaccine-7c1009a0bb0524b3,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-42e869aba90dc4bd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-a1a88d1c11b4fcdd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-d42b9fb3a44366ac,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-ce806d8b1a4132b2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.422743946314,higher,predicted +rcc-vaccine-ea51c18cc4e16995,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-9bce007ba3a4aff0,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-aa3019acbdddb7dc,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-1bcdaed331509a08,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.422743946314,higher,predicted +rcc-vaccine-4a1853322eea7a14,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-cde49959d75e6ec2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-7f8e1f8e02332663,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-b8aa4d97d214f765,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-5993797f7a6c7057,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-eacb2f310488ca9d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-70f05fcfa038224f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-4d15c4e62bb19311,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-a6b2180415ff1b56,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-e52f3609d694b4ba,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-763f29ff7ac4c65d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-c331f65899396b9b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-09c9b699d977f421,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-e944d5a782a7d074,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-75f86cf28dca60e2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-5b2ef450eaf2248a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-c50e7f0e1b3b69a5,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.973050177097,higher,predicted +rcc-vaccine-ae70f68909b113e8,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.597477078438,higher,predicted +rcc-vaccine-efdf21635c142ab8,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-817b1ac6328997a7,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-feb66d3c66c8a4de,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-7946e9a10fe114cf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-f3e32e857dd09d71,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-240bf5b232815c6a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-40c01a17a3ed4a2e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-7b823cc728ea1450,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-0c7eb5e3295f0ad2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-88aaf2311ce63196,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-bcae618cfad77c1f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.365942507982,higher,predicted +rcc-vaccine-11f61d47947266cc,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-f4a4db2362e7a54e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.577033996582,higher,predicted +rcc-vaccine-1eceb5b6a969d721,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-55106745d24cfbf4,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.356880515814,higher,predicted +rcc-vaccine-a808947689fbe42c,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.917404770851,higher,predicted +rcc-vaccine-590f5ea08f24e2cf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.893288612366,higher,predicted +rcc-vaccine-b984dc3be5c1fa17,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.577033996582,higher,predicted +rcc-vaccine-6ff6e758ac279425,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.85473293066,higher,predicted +rcc-vaccine-aa0d4e0503dacba4,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.729778409004,higher,predicted +rcc-vaccine-4d44829f31cf5558,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.862372934818,higher,predicted +rcc-vaccine-4f7d9f9f22d48440,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-60d1ba1b8ae350fd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.91576641798,higher,predicted +rcc-vaccine-7ec85836cb09ee30,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-4b7fdcc54f65b98b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.91576641798,higher,predicted +rcc-vaccine-d02de725ecc685a5,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.730106890202,higher,predicted +rcc-vaccine-e1d984ebf245728f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.892051696777,higher,predicted +rcc-vaccine-a9a3c2bc98a74746,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.473661601543,higher,predicted +rcc-vaccine-da610226e338a8de,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.803110301495,higher,predicted +rcc-vaccine-ca0a8a8bb4e2d094,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-c45585f253341070,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.964553833008,higher,predicted +rcc-vaccine-b2761351ccb757b7,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.473661601543,higher,predicted +rcc-vaccine-b1ad936ccc2f249a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.964553833008,higher,predicted +rcc-vaccine-02261377bb800592,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-834aff6d2a7dc363,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-0036b8c45a4cd20d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.473661601543,higher,predicted +rcc-vaccine-08447808dd0300dd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.964553833008,higher,predicted +rcc-vaccine-34c3e7e7a2dcd8b6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-1ae968c5ae01f49b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-29f7c92d094c1ced,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-0aaef5904890371c,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-27b47d43dab2b353,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-46b486b9d199e93a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-b8a9c44bcf53568f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.853769600391,higher,predicted +rcc-vaccine-7425139a6230adc0,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.795969843864,higher,predicted +rcc-vaccine-e39c49c350f42009,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.694881081581,higher,predicted +rcc-vaccine-73717d4619b66faf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-f130cbb5ea1ca0fa,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-3f16099c911d2791,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.795969843864,higher,predicted +rcc-vaccine-07bc8054e1edd65a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.558099448681,higher,predicted +rcc-vaccine-99bd603a759d6b22,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-ad726d8ef91bd0e6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-972e1f28fe4cb96a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.941862821579,higher,predicted +rcc-vaccine-e0d628668c1eccf2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-3d165d772065bb3a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-9ef49682993d2ea1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.355489432812,higher,predicted +rcc-vaccine-6c01a27a6a575007,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-12b152d35907e280,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.2018879354,higher,predicted diff --git a/results/raw_predictions/rcc/prime-2.0.csv b/results/raw_predictions/rcc/prime-2.0.csv new file mode 100644 index 0000000..e3ea1fc --- /dev/null +++ b/results/raw_predictions/rcc/prime-2.0.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,prime_percent_rank,mixmhcpred_percent_rank,allele +rcc-vaccine-8c2e1ec6d8f675cf,PRIME,2.0,immunogenicity,0.231349,higher,predicted,0.013,0.002,B5701 +rcc-vaccine-6a6ffb8b81ec9208,PRIME,2.0,immunogenicity,0.031043,higher,predicted,0.873,0.237,B4001 +rcc-vaccine-4adeedbe0e35cf09,PRIME,2.0,immunogenicity,,higher,unsupported_hla,,,C0602HLAA6601 +rcc-vaccine-e06799bdf8de8af5,PRIME,2.0,immunogenicity,0.231349,higher,predicted,0.013,0.002,B5701 +rcc-vaccine-969998ab9d8b4dc1,PRIME,2.0,immunogenicity,0.003103,higher,predicted,13.755,7.408,A6601 +rcc-vaccine-a5952e9e71bd27bd,PRIME,2.0,immunogenicity,0.021044,higher,predicted,1.546,1.758,B4001 +rcc-vaccine-6fa3e9ad450e030e,PRIME,2.0,immunogenicity,0.231349,higher,predicted,0.013,0.002,B5701 +rcc-vaccine-8a9041cc55b08e38,PRIME,2.0,immunogenicity,0.073447,higher,predicted,0.21,0.456,B4001 +rcc-vaccine-e11c81feedc5751e,PRIME,2.0,immunogenicity,0.000241,higher,predicted,68.331,20.277,A6601 +rcc-vaccine-fefffdcccca8649c,PRIME,2.0,immunogenicity,0.142751,higher,predicted,0.057,0.109,B4001 +rcc-vaccine-940b62df9c3078c4,PRIME,2.0,immunogenicity,0.173992,higher,predicted,0.041,0.075,A6601 +rcc-vaccine-43742d14e082c2e6,PRIME,2.0,immunogenicity,0.000881,higher,predicted,33.872,21.201,A6601 +rcc-vaccine-769c6b305827c570,PRIME,2.0,immunogenicity,0.01609,higher,predicted,2.513,0.657,B3503 +rcc-vaccine-e74f183589095f61,PRIME,2.0,immunogenicity,0.000172,higher,predicted,79.645,68.608,A0201 +rcc-vaccine-15ab70cb934aad8f,PRIME,2.0,immunogenicity,0.200487,higher,predicted,0.036,0.195,B3801 +rcc-vaccine-c8a8ce5ab15f9d4b,PRIME,2.0,immunogenicity,0.002497,higher,predicted,17.027,7.799,A0201 +rcc-vaccine-f05602883865f4bf,PRIME,2.0,immunogenicity,0.05337,higher,predicted,0.552,0.384,B3503 +rcc-vaccine-b92105a12fb6f028,PRIME,2.0,immunogenicity,0.211211,higher,predicted,0.03,0.025,B3801 +rcc-vaccine-a004075aed453bcf,PRIME,2.0,immunogenicity,0.002451,higher,predicted,16.927,2.108,B3501 +rcc-vaccine-70eb710f90e2973b,PRIME,2.0,immunogenicity,0.009664,higher,predicted,4.304,1.428,B3503 +rcc-vaccine-c81528933be4a147,PRIME,2.0,immunogenicity,0.231073,higher,predicted,0.013,0.04,A0101 +rcc-vaccine-2a2f0b8f959aa3f6,PRIME,2.0,immunogenicity,0.015484,higher,predicted,3.294,0.526,C0701 +rcc-vaccine-74e8a99b7ef86c61,PRIME,2.0,immunogenicity,0.099002,higher,predicted,0.199,0.162,B0801 +rcc-vaccine-88bea2527dd72a1d,PRIME,2.0,immunogenicity,0.0243,higher,predicted,2.529,5.059,C0602 +rcc-vaccine-ff6bbb8d69468a47,PRIME,2.0,immunogenicity,0.00482,higher,predicted,10.393,1.108,C0602 +rcc-vaccine-8ea6c4f21d2c81ee,PRIME,2.0,immunogenicity,0.01207,higher,predicted,3.638,0.788,B1302 +rcc-vaccine-b2f6110d808ac0da,PRIME,2.0,immunogenicity,0.108979,higher,predicted,0.141,0.117,A3001 +rcc-vaccine-f43e93564087f160,PRIME,2.0,immunogenicity,0.059356,higher,predicted,0.927,0.03,C0602 +rcc-vaccine-fc1276f53c3f45e1,PRIME,2.0,immunogenicity,0.00163,higher,predicted,21.154,11.829,A3001 +rcc-vaccine-3e6b1c1d8de0ce71,PRIME,2.0,immunogenicity,0.135059,higher,predicted,0.247,0.839,C0602 +rcc-vaccine-4255bdded9e984bd,PRIME,2.0,immunogenicity,0.14839,higher,predicted,0.16,0.078,C0701 +rcc-vaccine-31c3a5a1c962db43,PRIME,2.0,immunogenicity,0.105933,higher,predicted,0.174,0.157,B0801 +rcc-vaccine-d3d4046d155047b6,PRIME,2.0,immunogenicity,0.225538,higher,predicted,0.016,0.009,B1302 +rcc-vaccine-5e4096b57e93c92e,PRIME,2.0,immunogenicity,0.137711,higher,predicted,0.085,0.204,A0101 +rcc-vaccine-e8d6aeb51705642a,PRIME,2.0,immunogenicity,0.123093,higher,predicted,0.299,0.205,C0602 +rcc-vaccine-26dc24e186221fe2,PRIME,2.0,immunogenicity,0.276251,higher,predicted,0.003,0.026,A3001 +rcc-vaccine-364822c022710958,PRIME,2.0,immunogenicity,0.024145,higher,predicted,1.627,0.172,B0801 +rcc-vaccine-9c477d13a3359f5e,PRIME,2.0,immunogenicity,0.040185,higher,predicted,1.289,0.407,C0701 +rcc-vaccine-487a8e43d12c17ac,PRIME,2.0,immunogenicity,0.02398,higher,predicted,1.639,0.347,B0801 +rcc-vaccine-539b50b223ea0ee1,PRIME,2.0,immunogenicity,0.08652,higher,predicted,0.222,0.032,A0101 +rcc-vaccine-618d6aa7d8232657,PRIME,2.0,immunogenicity,0.012006,higher,predicted,4.131,2.584,C0701 +rcc-vaccine-71f5d2ab19a389e8,PRIME,2.0,immunogenicity,0.072737,higher,predicted,0.301,1.475,A0101 +rcc-vaccine-409a0661a319bbb1,PRIME,2.0,immunogenicity,0.040185,higher,predicted,1.289,0.407,C0701 +rcc-vaccine-9f431d040990f0f7,PRIME,2.0,immunogenicity,0.08679,higher,predicted,0.221,0.36,A0101 +rcc-vaccine-7c1009a0bb0524b3,PRIME,2.0,immunogenicity,0.00103,higher,predicted,29.367,4.658,C0701 +rcc-vaccine-42e869aba90dc4bd,PRIME,2.0,immunogenicity,0.002872,higher,predicted,13.901,11.529,A0101 +rcc-vaccine-a1a88d1c11b4fcdd,PRIME,2.0,immunogenicity,0.000868,higher,predicted,33.067,4.751,C0701 +rcc-vaccine-d42b9fb3a44366ac,PRIME,2.0,immunogenicity,0.08652,higher,predicted,0.222,0.032,A0101 +rcc-vaccine-ce806d8b1a4132b2,PRIME,2.0,immunogenicity,0.16414,higher,predicted,0.054,0.099,A0101 +rcc-vaccine-ea51c18cc4e16995,PRIME,2.0,immunogenicity,0.012006,higher,predicted,4.131,2.584,C0701 +rcc-vaccine-9bce007ba3a4aff0,PRIME,2.0,immunogenicity,0.002137,higher,predicted,18.037,8.154,C0701 +rcc-vaccine-aa3019acbdddb7dc,PRIME,2.0,immunogenicity,0.002911,higher,predicted,14.59,2.151,C0701 +rcc-vaccine-1bcdaed331509a08,PRIME,2.0,immunogenicity,0.16414,higher,predicted,0.054,0.099,A0101 +rcc-vaccine-4a1853322eea7a14,PRIME,2.0,immunogenicity,0.004955,higher,predicted,8.866,6.912,C0701 +rcc-vaccine-cde49959d75e6ec2,PRIME,2.0,immunogenicity,0.010927,higher,predicted,3.78,0.565,B0801 +rcc-vaccine-7f8e1f8e02332663,PRIME,2.0,immunogenicity,0.054186,higher,predicted,0.473,0.167,A0101 +rcc-vaccine-b8aa4d97d214f765,PRIME,2.0,immunogenicity,0.020849,higher,predicted,2.173,0.627,C0303 +rcc-vaccine-5993797f7a6c7057,PRIME,2.0,immunogenicity,0.024739,higher,predicted,1.31,0.764,A0301 +rcc-vaccine-eacb2f310488ca9d,PRIME,2.0,immunogenicity,0.045668,higher,predicted,0.474,0.461,B4001 +rcc-vaccine-70f05fcfa038224f,PRIME,2.0,immunogenicity,0.029104,higher,predicted,1.244,1.311,A3101 +rcc-vaccine-4d15c4e62bb19311,PRIME,2.0,immunogenicity,0.001505,higher,predicted,24.31,7.077,B5501 +rcc-vaccine-a6b2180415ff1b56,PRIME,2.0,immunogenicity,0.104449,higher,predicted,0.137,0.165,B5501 +rcc-vaccine-e52f3609d694b4ba,PRIME,2.0,immunogenicity,0.01842,higher,predicted,1.84,2.946,A0301 +rcc-vaccine-763f29ff7ac4c65d,PRIME,2.0,immunogenicity,0.016938,higher,predicted,1.982,2.051,B4001 +rcc-vaccine-c331f65899396b9b,PRIME,2.0,immunogenicity,0.059575,higher,predicted,0.579,0.238,C0303 +rcc-vaccine-09c9b699d977f421,PRIME,2.0,immunogenicity,0.078423,higher,predicted,0.278,0.423,A3101 +rcc-vaccine-e944d5a782a7d074,PRIME,2.0,immunogenicity,0.00665,higher,predicted,5.507,1.261,B4001 +rcc-vaccine-75f86cf28dca60e2,PRIME,2.0,immunogenicity,0.015183,higher,predicted,2.286,0.295,A0301 +rcc-vaccine-5b2ef450eaf2248a,PRIME,2.0,immunogenicity,0.023547,higher,predicted,1.908,0.479,C0303 +rcc-vaccine-c50e7f0e1b3b69a5,PRIME,2.0,immunogenicity,0.161118,higher,predicted,0.05,0.099,A0301 +rcc-vaccine-ae70f68909b113e8,PRIME,2.0,immunogenicity,0.122524,higher,predicted,0.079,0.728,B4001 +rcc-vaccine-efdf21635c142ab8,PRIME,2.0,immunogenicity,0.006965,higher,predicted,5.891,1,B5501 +rcc-vaccine-817b1ac6328997a7,PRIME,2.0,immunogenicity,0.016054,higher,predicted,2.784,0.379,A3201 +rcc-vaccine-feb66d3c66c8a4de,PRIME,2.0,immunogenicity,0.017987,higher,predicted,2.542,0.656,A0201 +rcc-vaccine-7946e9a10fe114cf,PRIME,2.0,immunogenicity,0.005165,higher,predicted,8.549,3.652,B4402 +rcc-vaccine-f3e32e857dd09d71,PRIME,2.0,immunogenicity,0.065268,higher,predicted,0.351,0.291,B4402 +rcc-vaccine-240bf5b232815c6a,PRIME,2.0,immunogenicity,0.189752,higher,predicted,0.039,0.109,C0501 +rcc-vaccine-40c01a17a3ed4a2e,PRIME,2.0,immunogenicity,0.022939,higher,predicted,1.927,0.689,A0201 +rcc-vaccine-7b823cc728ea1450,PRIME,2.0,immunogenicity,0.104224,higher,predicted,0.224,0.03,A3201 +rcc-vaccine-0c7eb5e3295f0ad2,PRIME,2.0,immunogenicity,0.027787,higher,predicted,1.291,0.125,B4402 +rcc-vaccine-88aaf2311ce63196,PRIME,2.0,immunogenicity,0.008837,higher,predicted,4.523,2.281,C0501 +rcc-vaccine-bcae618cfad77c1f,PRIME,2.0,immunogenicity,0.022536,higher,predicted,1.692,7.234,B4402 +rcc-vaccine-11f61d47947266cc,PRIME,2.0,immunogenicity,0.18811,higher,predicted,0.047,0.002,A3201 +rcc-vaccine-f4a4db2362e7a54e,PRIME,2.0,immunogenicity,0.201591,higher,predicted,0.04,0.049,A0201 +rcc-vaccine-1eceb5b6a969d721,PRIME,2.0,immunogenicity,0.038765,higher,predicted,0.852,0.241,B2705 +rcc-vaccine-55106745d24cfbf4,PRIME,2.0,immunogenicity,0.034507,higher,predicted,1.225,0.327,A0201 +rcc-vaccine-a808947689fbe42c,PRIME,2.0,immunogenicity,0.021044,higher,predicted,1.919,0.811,B2705 +rcc-vaccine-590f5ea08f24e2cf,PRIME,2.0,immunogenicity,0.029875,higher,predicted,1.254,1.004,B2705 +rcc-vaccine-b984dc3be5c1fa17,PRIME,2.0,immunogenicity,0.201591,higher,predicted,0.04,0.049,A0201 +rcc-vaccine-6ff6e758ac279425,PRIME,2.0,immunogenicity,0.060779,higher,predicted,0.559,1.106,A0201 +rcc-vaccine-aa0d4e0503dacba4,PRIME,2.0,immunogenicity,0.010168,higher,predicted,4.331,4.754,B2705 +rcc-vaccine-4d44829f31cf5558,PRIME,2.0,immunogenicity,0.070594,higher,predicted,0.343,0.943,B2705 +rcc-vaccine-4f7d9f9f22d48440,PRIME,2.0,immunogenicity,0.056781,higher,predicted,0.482,0.285,B2705 +rcc-vaccine-60d1ba1b8ae350fd,PRIME,2.0,immunogenicity,0.148328,higher,predicted,0.077,0.017,B2705 +rcc-vaccine-7ec85836cb09ee30,PRIME,2.0,immunogenicity,0.006543,higher,predicted,6.468,1.956,B2706 +rcc-vaccine-4b7fdcc54f65b98b,PRIME,2.0,immunogenicity,0.148328,higher,predicted,0.077,0.017,B2705 +rcc-vaccine-d02de725ecc685a5,PRIME,2.0,immunogenicity,0.116072,higher,predicted,0.136,0.078,B2705 +rcc-vaccine-e1d984ebf245728f,PRIME,2.0,immunogenicity,0.024212,higher,predicted,1.647,2.152,B2705 +rcc-vaccine-a9a3c2bc98a74746,PRIME,2.0,immunogenicity,0.134396,higher,predicted,0.079,0.206,B4402 +rcc-vaccine-da610226e338a8de,PRIME,2.0,immunogenicity,0.084327,higher,predicted,0.336,0.212,A0201 +rcc-vaccine-ca0a8a8bb4e2d094,PRIME,2.0,immunogenicity,0.004759,higher,predicted,8.165,3.166,C0501 +rcc-vaccine-c45585f253341070,PRIME,2.0,immunogenicity,0.271417,higher,predicted,0.005,0.008,B0801 +rcc-vaccine-b2761351ccb757b7,PRIME,2.0,immunogenicity,0.134396,higher,predicted,0.079,0.206,B4402 +rcc-vaccine-b1ad936ccc2f249a,PRIME,2.0,immunogenicity,0.271417,higher,predicted,0.005,0.008,B0801 +rcc-vaccine-02261377bb800592,PRIME,2.0,immunogenicity,0.010341,higher,predicted,4.173,0.359,B4402 +rcc-vaccine-834aff6d2a7dc363,PRIME,2.0,immunogenicity,0.028399,higher,predicted,1.251,0.374,B4402 +rcc-vaccine-0036b8c45a4cd20d,PRIME,2.0,immunogenicity,0.134396,higher,predicted,0.079,0.206,B4402 +rcc-vaccine-08447808dd0300dd,PRIME,2.0,immunogenicity,0.271417,higher,predicted,0.005,0.008,B0801 +rcc-vaccine-34c3e7e7a2dcd8b6,PRIME,2.0,immunogenicity,0.010341,higher,predicted,4.173,0.359,B4402 +rcc-vaccine-1ae968c5ae01f49b,PRIME,2.0,immunogenicity,0.028399,higher,predicted,1.251,0.374,B4402 +rcc-vaccine-29f7c92d094c1ced,PRIME,2.0,immunogenicity,0.054551,higher,predicted,0.469,0.085,B4402 +rcc-vaccine-0aaef5904890371c,PRIME,2.0,immunogenicity,0.028399,higher,predicted,1.251,0.374,B4402 +rcc-vaccine-27b47d43dab2b353,PRIME,2.0,immunogenicity,0.072271,higher,predicted,0.374,0.168,C0501 +rcc-vaccine-46b486b9d199e93a,PRIME,2.0,immunogenicity,0.078618,higher,predicted,0.525,0.319,C0701 +rcc-vaccine-b8a9c44bcf53568f,PRIME,2.0,immunogenicity,0.18383,higher,predicted,0.032,0.042,A0301 +rcc-vaccine-7425139a6230adc0,PRIME,2.0,immunogenicity,0.154046,higher,predicted,0.065,0.062,B1801 +rcc-vaccine-e39c49c350f42009,PRIME,2.0,immunogenicity,0.046887,higher,predicted,0.435,0.03,B0702 +rcc-vaccine-73717d4619b66faf,PRIME,2.0,immunogenicity,0.00442,higher,predicted,9.373,2.108,C0702 +rcc-vaccine-f130cbb5ea1ca0fa,PRIME,2.0,immunogenicity,0.034846,higher,predicted,1.486,1.968,C1203 +rcc-vaccine-3f16099c911d2791,PRIME,2.0,immunogenicity,0.154046,higher,predicted,0.065,0.062,B1801 +rcc-vaccine-07bc8054e1edd65a,PRIME,2.0,immunogenicity,0.123231,higher,predicted,0.071,0.161,B0702 +rcc-vaccine-99bd603a759d6b22,PRIME,2.0,immunogenicity,0.026807,higher,predicted,1.73,0.405,C0702 +rcc-vaccine-ad726d8ef91bd0e6,PRIME,2.0,immunogenicity,0.008608,higher,predicted,5.148,0.839,C0702 +rcc-vaccine-972e1f28fe4cb96a,PRIME,2.0,immunogenicity,0.053712,higher,predicted,0.499,0.324,B1801 +rcc-vaccine-e0d628668c1eccf2,PRIME,2.0,immunogenicity,0.015526,higher,predicted,4.37,1.203,A2501 +rcc-vaccine-3d165d772065bb3a,PRIME,2.0,immunogenicity,0.118113,higher,predicted,0.253,0.026,C1203 +rcc-vaccine-9ef49682993d2ea1,PRIME,2.0,immunogenicity,0.051028,higher,predicted,0.381,0.074,B0702 +rcc-vaccine-6c01a27a6a575007,PRIME,2.0,immunogenicity,0.049132,higher,predicted,0.842,0.272,C0702 +rcc-vaccine-12b152d35907e280,PRIME,2.0,immunogenicity,0.066229,higher,predicted,0.249,0.553,B0702 diff --git a/scripts/analyze_stability.py b/scripts/analyze_stability.py new file mode 100755 index 0000000..a8aea1c --- /dev/null +++ b/scripts/analyze_stability.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Descriptive cross-dataset stability and model-selection-risk analysis. + +This intentionally treats fixed predictors as descriptive/exploratory. It does +not fit models, test causal hypotheses, or relabel unsupported predictions. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import random +from collections import defaultdict +from itertools import combinations +from pathlib import Path + +KS = (1, 5, 10, 20) + + +def read_csv(p): + with Path(p).open(newline="", encoding="utf-8-sig") as f: + return list(csv.DictReader(f)) + + +def auc(rows): + x = sorted((float(r["score"]), int(r["label"])) for r in rows) + pos = sum(y for _, y in x) + neg = len(x) - pos + if not pos or not neg: + return None + rank = 0 + s = 0 + i = 0 + while i < len(x): + j = i + 1 + while j < len(x) and x[j][0] == x[i][0]: + j += 1 + rank = (i + j + 1) / 2 + s += rank * sum(y for _, y in x[i:j]) + i = j + return (s - pos * (pos + 1) / 2) / (pos * neg) + + +def patient_metric(rows, k): + groups = defaultdict(list) + for r in rows: + groups[r["patient_id"]].append(r) + vals = [] + for rs in groups.values(): + pos = sum(int(r["label"]) for r in rs) + if not pos: + continue + rs = sorted(rs, key=lambda r: -float(r["score"])) + top = rs[: min(k, len(rs))] + hits = sum(int(r["label"]) for r in top) + vals.append(hits / pos) + return sum(vals) / len(vals) if vals else None + + +def spearman(a, b): + def rank(v): + order = sorted(range(len(v)), key=v.__getitem__) + out = [0.0] * len(v) + i = 0 + while i < len(v): + j = i + 1 + while j < len(v) and v[order[j]] == v[order[i]]: + j += 1 + z = (i + j + 1) / 2 + for q in order[i:j]: + out[q] = z + i = j + return out + + if len(a) < 2: + return None + ra, rb = rank(a), rank(b) + ma = sum(ra) / len(ra) + mb = sum(rb) / len(rb) + num = sum((x - ma) * (y - mb) for x, y in zip(ra, rb)) + den = math.sqrt(sum((x - ma) ** 2 for x in ra) * sum((y - mb) ** 2 for y in rb)) + return num / den if den else None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--output-dir", type=Path, required=True) + ap.add_argument("--bootstrap", type=int, default=2000) + ap.add_argument("--seed", type=int, default=20260820) + ap.add_argument("--coverage", nargs="+", type=float, default=[0.5, 0.8, 0.95]) + ap.add_argument("--benchmark", nargs="+", required=True) + ap.add_argument("--predictions", nargs="+") + ap.add_argument("--prediction-dir", nargs="+") + args = ap.parse_args() + if args.prediction_dir and len(args.prediction_dir) != len(args.benchmark): + ap.error("prediction-dir count must match benchmark count") + if not args.prediction_dir and ( + not args.predictions or len(args.benchmark) != len(args.predictions) + ): + ap.error("provide aligned predictions or prediction-dir per benchmark") + datasets = {} + meta = {} + for idx, bp in enumerate(args.benchmark): + prediction_paths = ( + sorted(Path(args.prediction_dir[idx]).glob("*.csv")) + if args.prediction_dir + else [Path(args.predictions[idx])] + ) + for pp in prediction_paths: + b = {r["record_id"]: r for r in read_csv(bp)} + prs = read_csv(pp) + pred = prs[0]["predictor"] + rows = [] + status = defaultdict(int) + for p in prs: + status[p["status"]] += 1 + if p["status"] == "predicted": + r = b[p["record_id"]] + score = float(p["score"]) + score = score if p["score_direction"] == "higher" else -score + rows.append( + { + "record_id": p["record_id"], + "patient_id": r["patient_id"], + "study_id": r["study_id"], + "hla": r["hla"], + "label": int(r["immunogenicity"]), + "score": score, + } + ) + datasets.setdefault(Path(bp).stem, {})[pred] = rows + meta[pred] = { + "task": prs[0]["task"], + "version": prs[0]["predictor_version"], + "status": dict(status), + } + out = args.output_dir + out.mkdir(parents=True, exist_ok=True) + matrix = [] + rank_rows = [] + first_rows = [] + sens = [] + lodo = [] + domain_meta = [] + for ds, preds in datasets.items(): + supports = [{r["record_id"] for r in rs} for rs in preds.values()] + common = set.intersection(*supports) + benchmark_rows = read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds)) + domain_meta.append( + { + "dataset": ds, + "endpoint": benchmark_rows[0].get("assay_type", "unknown"), + "clinical_context": benchmark_rows[0].get("clinical_context", "unknown"), + "n_records": len(benchmark_rows), + "n_patients": len({r["patient_id"] for r in benchmark_rows}), + "n_studies": len({r["study_id"] for r in benchmark_rows}), + } + ) + for pred, rs in preds.items(): + rr = [r for r in rs if r["record_id"] in common] + matrix += [ + { + "dataset": ds, + "predictor": pred, + "metric": "AUROC", + "value": auc(rr), + "n": len(rr), + "coverage": len(rr) + / len(read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds))), + } + ] + for k in KS: + matrix.append( + { + "dataset": ds, + "predictor": pred, + "metric": f"Recall@{k}", + "value": patient_metric(rr, k), + "n": len(rr), + "coverage": len(rr) + / len(read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds))), + } + ) + study_ids = sorted({r["study_id"] for r in benchmark_rows}) + for held_out in study_ids: + for pred, rs in preds.items(): + rr = [r for r in rs if r["record_id"] in common and r["study_id"] != held_out] + lodo.append( + { + "dataset": ds, + "held_out_domain": held_out, + "predictor": pred, + "metric": "AUROC", + "value": auc(rr), + "n": len(rr), + "patients": len({r["patient_id"] for r in rr}), + "analysis_type": "descriptive_leave_one_domain_out", + } + ) + pairs = list(combinations(sorted(preds), 2)) + byid = {p: {r["record_id"]: r for r in preds[p]} for p in preds} + for a, bp in pairs: + ids = sorted(set(byid[a]) & set(byid[bp])) + ra = [byid[a][i] for i in ids] + rb = [byid[bp][i] for i in ids] + rank_rows.append( + { + "dataset": ds, + "left": a, + "right": bp, + "metric": "record_score_spearman", + "value": spearman([r["score"] for r in ra], [r["score"] for r in rb]), + "n": len(ids), + } + ) + for k in KS: + av = patient_metric(ra, k) + bv = patient_metric(rb, k) + rank_rows.append( + { + "dataset": ds, + "left": a, + "right": bp, + "metric": f"patient_Recall@{k}_difference", + "value": av - bv if av is not None and bv is not None else None, + "n": len(ids), + } + ) + patients = sorted({r["patient_id"] for r in next(iter(preds.values()))}) + rng = random.Random(args.seed) + wins = defaultdict(int) + reversals = defaultdict(int) + task_groups = defaultdict(list) + for p in preds: + task_groups[meta[p]["task"]].append(p) + for _ in range(args.bootstrap): + draw = rng.choices(patients, k=len(patients)) + vals = {} + for p, rs in preds.items(): + by = defaultdict(list) + for r in rs: + by[r["patient_id"]].append(r) + boot = [] + for i, pt in enumerate(draw): + boot += [{**r, "patient_id": f"{i}:{pt}"} for r in by[pt]] + vals[p] = patient_metric(boot, 5) or float("nan") + for task, group in task_groups.items(): + finite = {p: vals[p] for p in group if math.isfinite(vals[p])} + if finite: + top = max(finite.values()) + [ + wins.__setitem__((task, p), wins[(task, p)] + 1) + for p, v in finite.items() + if v == top + ] + if ( + "BigMHC" in vals + and "PRIME" in vals + and math.isfinite(vals["BigMHC"]) + and math.isfinite(vals["PRIME"]) + ): + reversals["BigMHC_vs_PRIME"] += int((vals["BigMHC"] - vals["PRIME"]) < 0) + for task, group in task_groups.items(): + for p in group: + first_rows.append( + { + "dataset": ds, + "task": task, + "predictor": p, + "metric": "Recall@5", + "probability_first": wins[(task, p)] / args.bootstrap, + } + ) + for key, v in reversals.items(): + sens.append( + { + "dataset": ds, + "comparison": key, + "metric": "probability_BigMHC_below_PRIME", + "value": v / args.bootstrap, + "strategy": "common_support", + "k": 5, + } + ) + for c in args.coverage: + for ds, preds in datasets.items(): + total = len(read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds))) + eligible = {p: rs for p, rs in preds.items() if len(rs) / total >= c} + eligible_supports = [{r["record_id"] for r in rs} for rs in eligible.values()] + union = set.intersection(*eligible_supports) if eligible_supports else set() + sens.append( + { + "dataset": ds, + "comparison": "coverage_filtered_predictors", + "metric": "common_support_n", + "value": len(union), + "strategy": f"coverage_threshold_{c}", + "predictors_in_support": "|".join(sorted(eligible)), + "k": 5, + } + ) + + def write(name, rows): + if not rows: + return + fields = sorted({k for r in rows for k in r}) + with (out / name).open("w", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields, lineterminator="\n") + w.writeheader() + w.writerows(rows) + + write("dataset_predictor_metric_matrix.csv", matrix) + write("rank_stability.csv", rank_rows) + write("model_selection_first_probability.csv", first_rows) + write("sensitivity_summary.csv", sens) + write("leave_one_domain_out.csv", lodo) + write("endpoint_domain_metadata.csv", domain_meta) + try: + import matplotlib.pyplot as plt + + labels = [f"{r['dataset']}\n{r['endpoint']}" for r in domain_meta] + names = sorted({r["predictor"] for r in matrix}) + vals = [] + for name in names: + vals.append( + [ + next( + ( + float(r["value"]) + for r in matrix + if r["dataset"] == d["dataset"] + and r["predictor"] == name + and r["metric"] == "AUROC" + ), + float("nan"), + ) + for d in domain_meta + ] + ) + fig, ax = plt.subplots(figsize=(8, 3.8)) + im = ax.imshow(vals, aspect="auto", vmin=0, vmax=1, cmap="viridis") + ax.set_yticks(range(len(names)), names) + ax.set_xticks(range(len(labels)), labels, rotation=20, ha="right") + ax.set_title("Exploratory AUROC by endpoint/domain and predictor") + fig.colorbar(im, ax=ax, label="AUROC") + fig.tight_layout() + fig.savefig(out / "endpoint_domain_auroc.png", dpi=180) + fig.savefig(out / "endpoint_domain_auroc.svg") + plt.close(fig) + except (ImportError, ModuleNotFoundError) as exc: + (out / "visualization_error.txt").write_text(f"matplotlib unavailable: {exc}\n") + (out / "analysis_metadata.json").write_text( + json.dumps( + { + "analysis_type": "exploratory_descriptive_heterogeneity", + "bootstrap": args.bootstrap, + "seed": args.seed, + "ks": KS, + "coverage_thresholds": args.coverage, + "datasets": list(datasets), + "predictors": meta, + "limitations": [ + "fixed pretrained scores; no causal inference", + "model-selection probabilities are conditional on observed patient samples", + "coverage strategies are descriptive and not multiplicity-adjusted", + ], + }, + indent=2, + ) + + "\n" + ) + print( + json.dumps( + { + "output_dir": str(out), + "datasets": list(datasets), + "matrix_rows": len(matrix), + "bootstrap": args.bootstrap, + } + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_release_boundary.py b/scripts/audit_release_boundary.py new file mode 100755 index 0000000..e4bbd51 --- /dev/null +++ b/scripts/audit_release_boundary.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Fail when a release tracks private environments, upstream sources, or raw inputs.""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +FORBIDDEN_PARTS = {".venv", "source", "vendor", "__pycache__"} +SECRET_PATTERN = re.compile( + rb"(?:sk-proj-[A-Za-z0-9_-]{20,}|sk-ant-[A-Za-z0-9_-]{20,}|" + rb"(?:OPENAI|ANTHROPIC)_API_KEY\s*=\s*[^\s]+)" +) + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + result = subprocess.run(["git", "ls-files", "-z"], cwd=root, capture_output=True, check=True) + tracked = [Path(item.decode("utf-8")) for item in result.stdout.split(b"\0") if item] + violations: list[str] = [] + for relative in tracked: + parts = set(relative.parts) + if FORBIDDEN_PARTS & parts: + violations.append(f"forbidden tracked directory: {relative}") + if relative.parts[:2] == ("data", "raw") and relative.name != ".gitkeep": + violations.append(f"raw input tracked: {relative}") + path = root / relative + if path.is_file() and path.stat().st_size <= 5_000_000: + try: + payload = path.read_bytes() + except OSError as error: + violations.append(f"unreadable tracked file: {relative}: {error}") + continue + if SECRET_PATTERN.search(payload): + violations.append(f"possible credential in tracked file: {relative}") + report = { + "status": "pass" if not violations else "fail", + "tracked_files": len(tracked), + "forbidden_parts": sorted(FORBIDDEN_PARTS), + "violations": violations, + } + print(json.dumps(report, indent=2, sort_keys=True)) + return 1 if violations else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_manuscript.py b/scripts/build_manuscript.py index 2e7e9b6..ff65379 100755 --- a/scripts/build_manuscript.py +++ b/scripts/build_manuscript.py @@ -79,6 +79,9 @@ def build_results(root: Path) -> tuple[str, str]: zhao_filtered = load_json(root / "data/zhao_vaccine_leakage_filter_summary.json") zhao_overlap = load_json(root / "research/training_overlap_summary_zhao.json") zhao_result = load_json(root / "results/analysis/zhao/fixed/metrics.json") + rcc_source = load_json(root / "data/rcc_vaccine_summary.json") + rcc_overlap = load_json(root / "research/training_overlap_summary_rcc.json") + rcc_result = load_json(root / "results/analysis/rcc/metrics.json") expanded_result = load_json(root / "results/analysis/improve/expanded_9_10/metrics.json") peptide_result = load_json(root / "results/analysis/improve/peptide_sensitivity/metrics.json") peptide_hla_rank_result = load_json( @@ -92,6 +95,7 @@ def build_results(root: Path) -> tuple[str, str]: fixed = load_csv(root / "results/tables/fixed_predictor_summary.csv") baselines = load_csv(root / "results/tables/heldout_baseline_summary.csv") hla = {row["predictor"]: row for row in load_csv(root / "results/analysis/improve/hla_sensitivity.csv")} + registry = load_csv(root / "data/predictor_registry.csv") benchmark_groups: dict[str, list[dict[str, str]]] = defaultdict(list) for row in load_csv(root / "data/processed/improve_benchmark.csv"): benchmark_groups[row["patient_id"]].append(row) @@ -121,8 +125,10 @@ def build_results(root: Path) -> tuple[str, str]: "baselines outperformed HLA-only baselines under both patient- and study-held-out fitting, " "while adding HLA to peptide features did not consistently improve over peptide features alone. " f"A frozen extension evaluated five models on {zhao_filtered['retained_rows']:,} overlap-filtered " - "vaccine peptides with a distinct post-vaccination ELISPOT endpoint. Support-matched random " - "ranking showed that high marginal Top-K values did not necessarily imply useful ranking signal." + "vaccine peptides with a distinct post-vaccination ELISPOT endpoint. A second endpoint-distinct " + f"vaccine cohort contributed {rcc_source['rows']} individually assayed short peptides from " + f"{rcc_source['patients']} patients. Support-matched and cross-domain analyses showed that high " + "marginal Top-K values did not necessarily imply stable or useful ranking signal." ) lopo = {row["predictor"]: row for row in baselines if row["analysis"] == "lopo"} @@ -154,6 +160,20 @@ def build_results(root: Path) -> tuple[str, str]: for row in zhao_result["paired_same_task"] if row["left"] == "BigMHC" and row["right"] == "PRIME" and row["metric"] == "ndcg@5" ) + rcc_rows = [] + rcc_benchmark = load_csv(root / "data/processed/rcc_vaccine_benchmark.csv") + for name, value in sorted(rcc_result["metrics"].items()): + random_ndcg5, _ = random_ranking_reference( + rcc_benchmark, load_csv(root / value["metadata"]["source"]), k=5 + ) + ci = value["patient_bootstrap_95ci"]["ndcg@5"] + rcc_rows.append( + f"| {name} | {value['pooled']['n']:,} | {f(value['pooled']['auroc'])} | " + f"{f(value['pooled']['average_precision'])} | {f(value['patient']['ndcg@5'])} " + f"({f(ci['low'])}–{f(ci['high'])}) | {f(random_ndcg5)} | " + f"{f(value['patient']['ndcg@5'] - random_ndcg5)} |" + ) + profile_only = sum(row["final_status"] != "reproduced" for row in registry) results = f"""### Public-artifact reproduction The version-pinned CPU workflows for MHCflurry 2.2.1, BigMHC v1.0 and PRIME 2.0 all produced complete outputs for the common benchmark. Reproduction nevertheless required tool-specific workarounds: MHCflurry model-path correction, a 4.6-GB BigMHC repository checkout and native rebuilding of PRIME and MixMHCpred binaries on Apple Silicon. These observations are recorded in `data/predictor_registry.csv`; they describe this platform and these pinned revisions rather than a universal installation-success rate. @@ -199,6 +219,22 @@ def build_results(root: Path) -> tuple[str, str]: | Predictor | Predicted records | AUROC | AP | Patient NDCG@5 (95% CI) | Random NDCG@5 | Gain over random | |---|---:|---:|---:|---:|---:|---:| {chr(10).join(zhao_rows)} + +### Endpoint-distinct RCC vaccine cohort + +The separately frozen RCC protocol retained {rcc_source['rows']} individually assayed short peptides from {rcc_source['patients']} vaccinated patients after excluding one source row with no usable short peptide/HLA assignment [@braun2025rcc]. The assay compared three peptide-stimulation replicates with three matched no-stimulation replicates; labels follow the source p-value threshold and therefore are assay-context outcomes rather than untreated biological negatives. No exact PRIME2, BigMHC-construction or DeepImmuno training overlap was identified among the {rcc_overlap['benchmark_rows']} records, while DeepHLApan row-level training identity remains unknown. + +On near-complete support, PRIME had AUROC {f(rcc_result['metrics']['PRIME']['pooled']['auroc'])} and patient NDCG@5 {f(rcc_result['metrics']['PRIME']['patient']['ndcg@5'])}; BigMHC had {f(rcc_result['metrics']['BigMHC']['pooled']['auroc'])} and {f(rcc_result['metrics']['BigMHC']['patient']['ndcg@5'])}, respectively. DeepImmuno-CNN supported only {rcc_result['metrics']['DeepImmuno-CNN']['pooled']['n']} records. With nine patients, all estimates are descriptive and do not establish a cross-domain interaction, universal ordering, natural tumour presentation or clinical efficacy. + +**Table 4. RCC personalized-vaccine cohort.** The endpoint is post-vaccination, individual-peptide IFN-γ ELISpot after in-vitro stimulation. Random NDCG@5 is calculated on each model's exact support. + +| Predictor | Predicted records | AUROC | AP | Patient NDCG@5 (95% CI) | Random NDCG@5 | Gain over random | +|---|---:|---:|---:|---:|---:|---:| +{chr(10).join(rcc_rows)} + +### Expanded reproducibility profile and extension contract + +The artifact census now records {len(registry)} pinned predictor entries. Beyond the five benchmarked tools, {profile_only} entries are retained as profile-only, non-comparable, pending or unreproducible outcomes rather than being silently omitted. The public extension contract supplies machine-validated Dataset Cards, Predictor Cards and prediction-artifact schemas together with a common-support evaluator; these additions improve reuse but do not make heterogeneous prediction tasks scientifically interchangeable. """ return abstract, results diff --git a/scripts/build_rcc_benchmark.py b/scripts/build_rcc_benchmark.py new file mode 100755 index 0000000..e1b608b --- /dev/null +++ b/scripts/build_rcc_benchmark.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Build the RCC vaccine cohort from the checksum-pinned Nature workbook.""" + +import argparse +import csv +import hashlib +import json +import re +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path + +EXPECTED = "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1" +AA = set("ACDEFGHIKLMNPQRSTVWY") +NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} + + +def text(v): + return "" if v is None else str(v).strip() + + +def norm_hla(v): + out = [] + for allele in re.split(r"[,;]", text(v)): + x = re.sub(r"[^A-Z0-9]", "", allele.upper().replace("HLA", "")) + m = re.fullmatch(r"([ABC])(\d{2})(\d{2})", x) + if not m: + raise ValueError(f"unsupported HLA {allele!r}") + out.append(f"HLA-{m.group(1)}*{m.group(2)}:{m.group(3)}") + return ";".join(out) + + +def rows(path): + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != EXPECTED: + raise ValueError("RCC member checksum mismatch") + z = zipfile.ZipFile(path) + ss = [] + root = ET.fromstring(z.read("xl/sharedStrings.xml")) + for si in root.findall("m:si", NS): + ss.append("".join(t.text or "" for t in si.findall(".//m:t", NS))) + root = ET.fromstring(z.read("xl/worksheets/sheet2.xml")) + table = [] + for ri, row in enumerate(root.findall(".//m:row", NS)): + vals = [] + for c in row.findall("m:c", NS): + v = c.find("m:v", NS) + val = "" if v is None else v.text + if c.get("t") == "s" and val: + val = ss[int(val)] + vals.append(val) + if ri == 0: + headers = vals + continue + d = dict(zip(headers, vals)) + pep = text(d["Short_Epitope"]).upper() + if pep.startswith("N/A") or text(d["HLA_of_best_short_epitope"]).upper() == "N/A": + continue + hla = norm_hla(d["HLA_of_best_short_epitope"]) + if not pep or set(pep) - AA or not 8 <= len(pep) <= 11: + raise ValueError(f"row {ri}: invalid peptide") + stim = [text(d[f"InVitro_PeptideStim_Replicate0{i}"]) for i in range(1, 4)] + nostim = [text(d[f"InVitro_NoStim_Replicate0{i}"]) for i in range(1, 4)] + if any(not x for x in stim + nostim): + raise ValueError(f"row {ri}: incomplete assay") + p = float(text(d["Ttest_pvalue_InVitroStim"])) + record = f"rcc-vaccine-{hashlib.sha256((text(d['Patient_ID']) + '|' + text(d['Peptide_ID']) + '|' + pep + '|' + hla).encode()).hexdigest()[:16]}" + table.append( + { + "record_id": record, + "patient_id": "RCC-" + text(d["Patient_ID"]), + "study_id": "RCC_PCV_VACCINE_2025", + "hla": hla, + "mhc_class": "I", + "peptide": pep, + "peptide_length": str(len(pep)), + "immunogenicity": str(int(p < 0.05)), + "label": str(int(p < 0.05)), + "assay_type": "post_vaccine_invitro_IFNG_ELISPOT", + "clinical_context": "personalized_RCC_peptide_vaccine", + "source_doi": "10.1038/s41586-024-08507-5", + "source_url": "https://doi.org/10.1038/s41586-024-08507-5", + "source_checksum": EXPECTED, + "source_row": str(ri + 1), + "source_pvalue": text(d["Ttest_pvalue_InVitroStim"]), + "evidence_level": "individual_peptide_stimulation_with_matched_no_stimulation_control", + } + ) + return table + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--input", type=Path, required=True) + ap.add_argument("--output", type=Path, required=True) + ap.add_argument("--summary", type=Path, required=True) + a = ap.parse_args() + data = rows(a.input) + assert len(data) == 129 and len({r["patient_id"] for r in data}) == 9 + a.output.parent.mkdir(parents=True, exist_ok=True) + a.summary.parent.mkdir(parents=True, exist_ok=True) + with a.output.open("w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(data[0]), lineterminator="\n") + w.writeheader() + w.writerows(data) + summary = { + "rows": len(data), + "patients": len({r["patient_id"] for r in data}), + "positives": sum(int(r["label"]) for r in data), + "negatives": sum(not int(r["label"]) for r in data), + "member_sha256": EXPECTED, + "endpoint": "post-vaccine individual-peptide IFNG ELISpot", + "hla_semantics": "predicted best short-epitope binding allele", + } + a.summary.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(json.dumps(summary, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_results_manifest.py b/scripts/build_results_manifest.py index e1c4992..812f3c0 100755 --- a/scripts/build_results_manifest.py +++ b/scripts/build_results_manifest.py @@ -86,6 +86,41 @@ "reports/improve_expanded_predictor_run.json", "reports/new_predictor_determinism.json", "reports/extension_clean_reproduction.json", + "data/processed/rcc_vaccine_benchmark.csv", + "data/processed/rcc_vaccine_benchmark_full.csv", + "data/rcc_vaccine_summary.json", + "research/extension_protocol_rcc_v1.json", + "research/training_overlap_audit_rcc.csv", + "research/training_overlap_summary_rcc.json", + "results/raw_predictions/rcc/bigmhc-v1.0.csv", + "results/raw_predictions/rcc/prime-2.0.csv", + "results/raw_predictions/rcc/deepimmuno-cnn.csv", + "results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "reports/rcc_predictor_run.json", + "results/analysis/rcc/metrics.json", + "results/analysis/rcc/missingness.csv", + "results/analysis/rcc/paired_differences.csv", + "results/analysis/stability/analysis_metadata.json", + "results/analysis/stability/dataset_predictor_metric_matrix.csv", + "results/analysis/stability/rank_stability.csv", + "results/analysis/stability/model_selection_first_probability.csv", + "results/analysis/stability/sensitivity_summary.csv", + "results/analysis/stability/leave_one_domain_out.csv", + "results/analysis/stability/endpoint_domain_metadata.csv", + "results/analysis/stability/endpoint_domain_auroc.svg", + "results/analysis/stability/endpoint_domain_auroc.png", + "contracts/dataset-card.example.json", + "contracts/predictor-card.example.json", + "contracts/prediction-artifact.schema.json", + "contracts/synthetic/benchmark.csv", + "contracts/synthetic/predictions.csv", + "predictors/mhcmatch/attempts/sweep_receipt.json", + "predictors/mhcnuggets/attempts/sweep_receipt.json", + "predictors/neofox/attempts/sweep_receipt.json", + "predictors/neoguider/attempts/sweep_receipt.json", + "predictors/pvactools/attempts/sweep_receipt.json", + "predictors/seq2neo/attempts/sweep_receipt.json", + "predictors/vaxrank/attempts/sweep_receipt.json", ) diff --git a/scripts/evaluate_benchmark.py b/scripts/evaluate_benchmark.py index 40b1092..2abf6a6 100755 --- a/scripts/evaluate_benchmark.py +++ b/scripts/evaluate_benchmark.py @@ -426,7 +426,9 @@ def main() -> int: json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n" ) with (args.output_dir / "missingness.csv").open("w", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["predictor", "status", "count"]) + writer = csv.DictWriter( + handle, fieldnames=["predictor", "status", "count"], lineterminator="\n" + ) writer.writeheader() writer.writerows(missingness) with (args.output_dir / "paired_differences.csv").open("w", newline="") as handle: @@ -444,7 +446,7 @@ def main() -> int: "ci_low", "ci_high", ] - writer = csv.DictWriter(handle, fieldnames=fields) + writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") writer.writeheader() writer.writerows(paired) print( diff --git a/scripts/render_stability_figure.py b/scripts/render_stability_figure.py new file mode 100755 index 0000000..e9fa23f --- /dev/null +++ b/scripts/render_stability_figure.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Render a dependency-free SVG from the stability matrix.""" +import csv +import html +from pathlib import Path + +root=Path(__file__).resolve().parents[1]; out=root/"results/analysis/stability"; rows=list(csv.DictReader((out/"dataset_predictor_metric_matrix.csv").open())) +ds=sorted({r["dataset"] for r in rows}); names=sorted({r["predictor"] for r in rows}); W,H=900,100+len(names)*34; left,top,cell=220,55,110 +def value(d,n): + for r in rows: + if r["dataset"]==d and r["predictor"]==n and r["metric"]=="AUROC": return float(r["value"]) + return None +def color(v): + if v is None:return "#d9d9d9" + q=max(0,min(1,v)); return f"rgb({int(245-180*q)},{int(245-120*q)},{int(245-20*q)})" +s=[f'', '', 'Exploratory AUROC by endpoint/domain and predictor'] +for j,d in enumerate(ds): s += [f'{html.escape(d)}'] +for i,n in enumerate(names): + y=top+i*34; s.append(f'{html.escape(n)}') + for j,d in enumerate(ds): + v=value(d,n); x=left+j*cell; s.append(f''); s.append(f'{"NA" if v is None else f"{v:.3f}"}') +s += [f'Fixed pretrained scores; descriptive heterogeneity only; endpoint/domain differences are not causal estimates.',''] +(out/"endpoint_domain_auroc.svg").write_text("\n".join(s)+"\n") diff --git a/scripts/reproduce_public_predictors.py b/scripts/reproduce_public_predictors.py new file mode 100755 index 0000000..350ad8f --- /dev/null +++ b/scripts/reproduce_public_predictors.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Cheap, auditable first-pass reproduction sweep for public predictors. + +This deliberately stops at installation/import/CLI smoke tests. It never +silently converts an end-to-end workflow into a peptide-HLA benchmark model. +""" + +from __future__ import annotations + +import argparse +import json +import platform +import shutil +import subprocess +import time +from pathlib import Path + +CANDIDATES = { + "mhcnuggets": "https://github.com/KarchinLab/mhcnuggets.git", + "neoguider": "https://github.com/XuegongLab/neoguider.git", + "neofox": "https://github.com/TRON-Bioinformatics/neofox.git", + "pvactools": "https://github.com/griffithlab/pVACtools.git", + "seq2neo": "https://github.com/XSLiuLab/Seq2Neo.git", + "vaxrank": "https://github.com/openvax/vaxrank.git", + "mhcmatch": "https://github.com/antigenomics/mhcmatch.git", +} +IMPORTS = {"mhcnuggets": "mhcnuggets", "neofox": "neofox", "vaxrank": "vaxrank"} +CLI = {"pvactools": "pvacseq", "vaxrank": "vaxrank"} + + +def run(cmd: list[str], cwd: Path | None = None, timeout: int = 180): + t = time.monotonic() + try: + p = subprocess.run( + cmd, cwd=cwd, text=True, capture_output=True, timeout=timeout, check=False + ) + return p.returncode, p.stdout, p.stderr, time.monotonic() - t + except subprocess.TimeoutExpired as e: + return ( + 124, + e.stdout or "", + (e.stderr or "") + f"\nTimed out after {timeout}s", + time.monotonic() - t, + ) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--only", nargs="*", choices=sorted(CANDIDATES), default=sorted(CANDIDATES)) + args = ap.parse_args() + root = Path(__file__).resolve().parents[1] + uv = shutil.which("uv") + rows = [] + for name in args.only: + base = root / "predictors" / name + source = base / "source" + attempts = base / "attempts" + attempts.mkdir(parents=True, exist_ok=True) + if not source.exists(): + rc, out, err, _sec = run( + ["git", "clone", "--filter=blob:none", CANDIDATES[name], str(source)], root, 300 + ) + (attempts / "clone.stdout.log").write_text(out) + (attempts / "clone.stderr.log").write_text(err) + rc, rev, err, _sec = run(["git", "rev-parse", "HEAD"], source) + revision = rev.strip() if rc == 0 else "unknown" + license_files = sorted( + str(p.relative_to(source)) + for p in source.rglob("*") + if p.is_file() + and ("license" in p.name.lower() or p.name.lower() in {"copying", "notice"}) + )[:20] + env = base / ".venv" + install_rc = 125 + install_out = install_err = "not attempted" + if uv and source.exists(): + if not env.exists(): + run([uv, "venv", "--python", "3.11", str(env)], root, 180) + py = str(env / "bin/python") + install_rc, install_out, install_err, _ = run( + [uv, "pip", "install", "--python", py, "-e", str(source)], root, 900 + ) + (attempts / "install.stdout.log").write_text(install_out or "") + (attempts / "install.stderr.log").write_text(install_err or "") + smoke_rc = 125 + smoke_out = smoke_err = "not attempted" + if install_rc == 0 and name in IMPORTS: + smoke_rc, smoke_out, smoke_err, _ = run( + [str(env / "bin/python"), "-c", f"import {IMPORTS[name]}; print('import ok')"], + root, + 180, + ) + elif install_rc == 0 and name in CLI: + smoke_rc, smoke_out, smoke_err, _ = run( + [str(env / ("bin/" + CLI[name])), "--help"], root, 180 + ) + (attempts / "smoke.stdout.log").write_text(smoke_out or "") + (attempts / "smoke.stderr.log").write_text(smoke_err or "") + receipt = { + "predictor": name, + "revision": revision, + "repo": CANDIDATES[name], + "install_returncode": install_rc, + "smoke_returncode": smoke_rc, + "license_files": license_files, + "platform": platform.platform(), + "benchmark_track": "profile_only", + } + (attempts / "sweep_receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + rows.append((name, revision, install_rc, smoke_rc, ";".join(license_files))) + print("predictor\trevision\tinstall_rc\tsmoke_rc\tlicense_files") + for row in rows: + print("\t".join(map(str, row))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_external_cohort_funnel.py b/scripts/validate_external_cohort_funnel.py new file mode 100755 index 0000000..e907eba --- /dev/null +++ b/scripts/validate_external_cohort_funnel.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Deterministic validation for the external-cohort screening funnel.""" + +import csv +import json +import pathlib + +root = pathlib.Path(__file__).resolve().parents[1] +csv_path = root / "research" / "external_cohort_funnel.csv" +json_path = root / "research" / "external_cohort_failure_protocols.json" +allowed = {"eligible", "pending", "failed"} +required = {"candidate_id", "eligibility", "decision", "evidence_note", "source_url"} +with csv_path.open(newline="", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + assert reader.fieldnames is not None + rows = list(reader) +assert all(None not in row for row in rows), "malformed CSV row" +assert rows and required <= set(rows[0]), "missing funnel columns" +ids = {r["candidate_id"] for r in rows} +assert len(ids) == len(rows), "duplicate candidate_id" +for row in rows: + assert row["eligibility"] in allowed, row + assert row["decision"] in { + "existing_primary", + "existing_external", + "eligible_external", + "failed_eligibility", + "failed_leakage_gate", + "pending", + "failed", + }, row + assert row["source_url"].startswith(("http://", "https://")), row +data = json.loads(json_path.read_text(encoding="utf-8")) +failure_ids = {x["candidate_id"] for x in data["failures"]} +assert failure_ids <= ids +pending_ids = {x["candidate_id"] for x in data["pending"]} +assert pending_ids <= ids +assert not failure_ids & pending_ids +eligible_ids = {x["candidate_id"] for x in data["eligible_after_member_audit"]} +assert eligible_ids == { + r["candidate_id"] + for r in rows + if r["eligibility"] == "eligible" and r["decision"] == "eligible_external" +} +print(f"validated {len(rows)} candidates: {len(failure_ids)} failures, {len(pending_ids)} pending") diff --git a/src/neorepro/cli.py b/src/neorepro/cli.py index 7e7bd0c..f776cb7 100644 --- a/src/neorepro/cli.py +++ b/src/neorepro/cli.py @@ -8,10 +8,21 @@ from pathlib import Path from neorepro.audit import AuditError, audit_predictions +from neorepro.contract import ( + ContractError, + evaluate, + markdown_report, + validate_artifact, + validate_card, +) def project_root() -> Path: - return Path.cwd() + cwd = Path.cwd() + if (cwd / "RESEARCH_SPEC.md").exists(): + return cwd + # Keep the legacy audit usable when invoked from another working directory. + return Path(__file__).resolve().parents[2] def list_predictors(root: Path) -> int: @@ -54,6 +65,39 @@ def main(argv: list[str] | None = None) -> int: ), ) audit_parser.add_argument("predictions", type=Path, help="five-column prediction CSV") + + def add_path_command(name, help_text): + p = subparsers.add_parser(name, help=help_text) + p.add_argument("path", type=Path) + return p + + dataset_parser = subparsers.add_parser("dataset", help="validate a Dataset Card JSON") + dataset_parser.add_argument("action", choices=["validate"]) + dataset_parser.add_argument("path", type=Path) + predictor_parser = subparsers.add_parser("predictor", help="validate a Predictor Card JSON") + predictor_parser.add_argument("action", choices=["validate"]) + predictor_parser.add_argument("path", type=Path) + artifact_parser = add_path_command("artifact", "validate a prediction artifact CSV") + artifact_parser.add_argument("--benchmark", type=Path) + overlap_parser = add_path_command( + "overlap-audit", "run the frozen legacy overlap/common-support audit" + ) + overlap_parser.add_argument("--root", type=Path, default=None) + evaluate_parser = subparsers.add_parser("evaluate", help="evaluate artifacts on common support") + evaluate_parser.add_argument("benchmark", type=Path) + evaluate_parser.add_argument("artifacts", nargs="+", type=Path) + evaluate_parser.add_argument("--overlap-audit", type=Path) + evaluate_parser.add_argument( + "--output", type=Path, default=Path("results/contract_evaluation.json") + ) + evaluate_parser.add_argument( + "--report", type=Path, default=Path("reports/contract_evaluation.md") + ) + report_parser = subparsers.add_parser("report", help="render an evaluation JSON as Markdown") + report_parser.add_argument("evaluation", type=Path) + report_parser.add_argument( + "--output", type=Path, default=Path("reports/contract_evaluation.md") + ) args = parser.parse_args(argv) root = project_root() if args.command == "list-predictors": @@ -67,6 +111,30 @@ def main(argv: list[str] | None = None) -> int: parser.error(str(error)) print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) return 0 + try: + if args.command == "dataset": + print(json.dumps(validate_card(args.path, "dataset"), indent=2)) + return 0 + if args.command == "predictor": + print(json.dumps(validate_card(args.path, "predictor"), indent=2)) + return 0 + if args.command == "artifact": + print(json.dumps(validate_artifact(args.path, args.benchmark), indent=2)) + return 0 + if args.command == "overlap-audit": + report = audit_predictions(args.path, args.root or root) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + if args.command == "evaluate": + result = evaluate(args.benchmark, args.artifacts, args.output, args.overlap_audit) + markdown_report(result, args.report) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + if args.command == "report": + markdown_report(json.loads(args.evaluation.read_text()), args.output) + return 0 + except (ContractError, OSError, KeyError, ValueError) as error: + parser.error(str(error)) raise AssertionError(args.command) diff --git a/src/neorepro/contract.py b/src/neorepro/contract.py new file mode 100644 index 0000000..19d5fb9 --- /dev/null +++ b/src/neorepro/contract.py @@ -0,0 +1,211 @@ +"""Public extension contract: cards, artifacts, gates, evaluation and reports.""" + +from __future__ import annotations + +import csv +import json +import math +from collections import defaultdict +from pathlib import Path + +from neorepro.metrics import auroc, tie_aware_ranking_metrics + +DATASET_REQUIRED = { + "dataset_id", + "version", + "records_path", + "label_column", + "patient_id_column", + "score_tasks", +} +PREDICTOR_REQUIRED = {"predictor_id", "version", "task", "score_direction", "adapter", "license"} +ARTIFACT_REQUIRED = { + "record_id", + "predictor", + "predictor_version", + "task", + "score", + "score_direction", + "status", +} +STATUSES = {"predicted", "unsupported", "failed", "invalid"} + + +class ContractError(ValueError): + pass + + +def _json(path): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + raise ContractError(f"invalid JSON: {path}: {e}") from e + + +def validate_card(path: Path, kind: str) -> dict: + if kind not in {"dataset", "predictor"}: + raise ContractError(f"unknown card kind: {kind}") + obj = _json(path) + required = DATASET_REQUIRED if kind == "dataset" else PREDICTOR_REQUIRED + missing = sorted(required - obj.keys()) + if missing: + raise ContractError(f"{kind} card missing fields: {', '.join(missing)}") + if not isinstance(obj["version"], str) or not obj["version"]: + raise ContractError("card version must be non-empty") + if kind == "predictor" and obj["score_direction"] not in {"higher", "lower"}: + raise ContractError("score_direction must be higher or lower") + return { + "valid": True, + "kind": kind, + "path": str(path), + "id": obj.get("dataset_id", obj.get("predictor_id")), + "version": obj["version"], + } + + +def read_csv(path): + with Path(path).open(newline="", encoding="utf-8-sig") as f: + r = csv.DictReader(f) + if not r.fieldnames: + raise ContractError(f"CSV has no header: {path}") + return list(r), list(r.fieldnames) + + +def validate_artifact(path: Path, benchmark: Path | None = None) -> dict: + rows, header = read_csv(path) + missing = sorted(ARTIFACT_REQUIRED - set(header)) + if missing: + raise ContractError(f"prediction artifact missing columns: {', '.join(missing)}") + if not rows: + raise ContractError("prediction artifact has no rows") + ids = set() + problems = [] + for n, row in enumerate(rows, 2): + if not row["record_id"] or row["record_id"] in ids: + problems.append(f"line {n}: duplicate/blank record_id") + ids.add(row["record_id"]) + if row["status"] not in STATUSES: + problems.append(f"line {n}: unknown status {row['status']}") + if row["score_direction"] not in {"higher", "lower"}: + problems.append(f"line {n}: invalid score_direction") + if row["status"] == "predicted": + try: + if not math.isfinite(float(row["score"])): + raise ValueError + except ValueError: + problems.append(f"line {n}: predicted score must be finite") + if benchmark: + brows, _ = read_csv(benchmark) + expected = {r["record_id"] for r in brows} + if ids != expected: + problems.append( + f"record support mismatch: submitted={len(ids)} expected={len(expected)}" + ) + if problems: + raise ContractError("; ".join(problems[:8])) + return { + "valid": True, + "rows": len(rows), + "predictors": sorted({r["predictor"] for r in rows}), + "tasks": sorted({r["task"] for r in rows}), + "predicted": sum(r["status"] == "predicted" for r in rows), + "missing": sum(r["status"] != "predicted" for r in rows), + } + + +def evaluate( + benchmark: Path, artifacts: list[Path], output: Path, overlap_audit: Path | None = None +) -> dict: + b, _ = read_csv(benchmark) + labels = {r["record_id"]: int(r["label"] if "label" in r else r["immunogenicity"]) for r in b} + patients = {r["record_id"]: r.get("patient_id", "unknown") for r in b} + result = { + "schema_version": 1, + "benchmark": str(benchmark), + "models": {}, + "gates": {"common_support": True, "missingness": True, "leakage": "not_checked"}, + } + if overlap_audit: + audit_rows, audit_header = read_csv(overlap_audit) + if "record_id" not in audit_header: + raise ContractError("overlap audit must contain record_id") + audit_ids = {r["record_id"] for r in audit_rows} + if not audit_ids <= set(labels): + raise ContractError("overlap audit contains record_id absent from benchmark") + result["gates"]["leakage"] = { + "status": "checked", + "audit": str(overlap_audit), + "rows": len(audit_rows), + } + parsed = [] + for p in artifacts: + validate_artifact(p, benchmark) + rows, _ = read_csv(p) + parsed.append(rows) + supports = [{r["record_id"] for r in rows if r["status"] == "predicted"} for rows in parsed] + common = set.intersection(*supports) if supports else set() + result["common_support"] = { + "records": len(common), + "coverage": len(common) / len(labels) if labels else 0, + } + for rows, support in zip(parsed, supports): + name = rows[0]["predictor"] + scores = {r["record_id"]: float(r["score"]) for r in rows if r["status"] == "predicted"} + use = common + direction = rows[0]["score_direction"] + vals = [scores[k] if direction == "higher" else -scores[k] for k in use] + labs = [labels[k] for k in use] + model = { + "rows": len(rows), + "predicted": len(support), + "missing": len(set(labels) - support), + "coverage": len(support) / len(labels), + "task": rows[0]["task"], + "score_direction": direction, + } + if len(set(labs)) == 2: + model["auroc"] = auroc(labs, vals) + by = defaultdict(list) + for k in use: + by[patients[k]].append(k) + pm = [] + for ks in by.values(): + if any(labels[k] for k in ks): + pm.append( + tie_aware_ranking_metrics( + [labels[k] for k in ks], + [scores[k] if direction == "higher" else -scores[k] for k in ks], + [5], + ) + ) + if pm: + model["patient_ndcg@5"] = sum(x["ndcg@5"] for x in pm) / len(pm) + result["models"][name] = model + output.parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return result + + +def markdown_report(result: dict, output: Path): + lines = [ + "# NeoRepro evaluation report", + "", + f"Common support: **{result['common_support']['records']}** records ({result['common_support']['coverage']:.1%}).", + "", + "| Predictor | Task | Coverage | AUROC | Patient NDCG@5 |", + "|---|---|---:|---:|---:|", + ] + for name, m in result["models"].items(): + lines.append( + f"| {name} | {m['task']} | {m['coverage']:.1%} | {m.get('auroc', 'unknown')} | {m.get('patient_ndcg@5', 'unknown')} |" + ) + lines += [ + "", + "## Gates", + "", + "- Common support: passed for reported comparisons.", + "- Missingness: reported per predictor; no failed row was imputed.", + "- Leakage: run `neorepro overlap-audit`; unknown training overlap remains unknown.", + ] + output.parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000..9fc9516 --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import pytest + +from neorepro.contract import ContractError, evaluate, validate_artifact, validate_card + +ROOT = Path(__file__).resolve().parents[1] + + +def test_cards_and_artifact(): + assert validate_card(ROOT / "contracts/dataset-card.example.json", "dataset")["valid"] + assert validate_card(ROOT / "contracts/predictor-card.example.json", "predictor")["valid"] + assert ( + validate_artifact( + ROOT / "contracts/synthetic/predictions.csv", ROOT / "contracts/synthetic/benchmark.csv" + )["missing"] + == 1 + ) + + +def test_evaluate_synthetic(tmp_path): + out = tmp_path / "eval.json" + result = evaluate( + ROOT / "contracts/synthetic/benchmark.csv", + [ROOT / "contracts/synthetic/predictions.csv"], + out, + ) + assert result["common_support"]["records"] == 3 + assert result["models"]["synthetic"]["auroc"] == 1.0 + + +def test_lower_direction_and_overlap_gate(tmp_path): + lower = """record_id,predictor,predictor_version,task,score,score_direction,status +s1,synthetic,1.0,immunogenicity,0.1,lower,predicted +s2,synthetic,1.0,immunogenicity,0.9,lower,predicted +s3,synthetic,1.0,immunogenicity,0.2,lower,predicted +s4,synthetic,1.0,immunogenicity,,lower,unsupported +""" + path = tmp_path / "lower.csv" + path.write_text(lower) + audit = tmp_path / "overlap.csv" + audit.write_text("record_id,exact_overlap\ns1,0\ns2,0\ns3,0\ns4,0\n") + result = evaluate( + ROOT / "contracts/synthetic/benchmark.csv", [path], tmp_path / "out.json", audit + ) + assert result["models"]["synthetic"]["score_direction"] == "lower" + assert result["models"]["synthetic"]["auroc"] == 1.0 + assert result["gates"]["leakage"]["status"] == "checked" + + +def test_artifact_rejects_duplicate_or_missing_support(tmp_path): + rows = (ROOT / "contracts/synthetic/predictions.csv").read_text().splitlines() + path = tmp_path / "bad.csv" + path.write_text("\n".join(rows[:-1] + [rows[1]]) + "\n") + with pytest.raises(ContractError, match="duplicate|support mismatch"): + validate_artifact(path, ROOT / "contracts/synthetic/benchmark.csv") diff --git a/tests/test_rcc_extension.py b/tests/test_rcc_extension.py new file mode 100644 index 0000000..84a665d --- /dev/null +++ b/tests/test_rcc_extension.py @@ -0,0 +1,41 @@ +import csv +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def load_csv(path: str) -> list[dict[str, str]]: + with (ROOT / path).open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def test_rcc_benchmark_and_overlap_contract() -> None: + rows = load_csv("data/processed/rcc_vaccine_benchmark.csv") + assert len(rows) == 129 + assert len({row["patient_id"] for row in rows}) == 9 + assert sum(int(row["immunogenicity"]) for row in rows) == 75 + summary = json.loads((ROOT / "research/training_overlap_summary_rcc.json").read_text()) + assert summary["benchmark_union_known_exact_overlap"] == 0 + assert summary["overlap_dimensions"]["deephlapan_training_identity"].startswith("unknown_") + + +def test_rcc_predictions_are_complete_artifacts() -> None: + paths = sorted((ROOT / "results/raw_predictions/rcc").glob("*.csv")) + assert len(paths) == 4 + for path in paths: + rows = load_csv(str(path.relative_to(ROOT))) + assert len(rows) == 129 + assert len({row["record_id"] for row in rows}) == 129 + assert {row["status"] for row in rows} <= { + "predicted", + "unsupported_hla", + "unsupported_length", + } + + +def test_generated_manuscript_contains_rcc_table() -> None: + manuscript = (ROOT / "paper/manuscript_resource.md").read_text(encoding="utf-8") + assert "### Endpoint-distinct RCC vaccine cohort" in manuscript + assert "**Table 4. RCC personalized-vaccine cohort.**" in manuscript + assert "[@braun2025rcc]" in manuscript diff --git a/tests/test_stability_analysis.py b/tests/test_stability_analysis.py new file mode 100644 index 0000000..b4e65e8 --- /dev/null +++ b/tests/test_stability_analysis.py @@ -0,0 +1,48 @@ +import csv +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "results/analysis/stability" + + +class StabilityAnalysisTest(unittest.TestCase): + def test_stability_outputs_have_required_artifacts(self): + required = [ + "dataset_predictor_metric_matrix.csv", + "rank_stability.csv", + "model_selection_first_probability.csv", + "sensitivity_summary.csv", + "leave_one_domain_out.csv", + "endpoint_domain_metadata.csv", + "analysis_metadata.json", + ] + self.assertTrue(all((OUT / name).exists() for name in required)) + meta = json.loads((OUT / "analysis_metadata.json").read_text()) + self.assertEqual(meta["analysis_type"], "exploratory_descriptive_heterogeneity") + self.assertEqual(meta["bootstrap"], 2000) + self.assertEqual( + set(meta["datasets"]), + { + "improve_benchmark", + "zhao_vaccine_benchmark", + "rcc_vaccine_benchmark", + }, + ) + + def test_task_specific_first_probabilities_sum_to_one(self): + rows = list(csv.DictReader((OUT / "model_selection_first_probability.csv").open())) + for key in {(r["dataset"], r["task"], r["metric"]) for r in rows}: + group = [ + float(r["probability_first"]) + for r in rows + if (r["dataset"], r["task"], r["metric"]) == key + ] + self.assertAlmostEqual(sum(group), 1.0) + + def test_lodo_is_explicitly_descriptive(self): + rows = list(csv.DictReader((OUT / "leave_one_domain_out.csv").open())) + self.assertTrue( + rows and {r["analysis_type"] for r in rows} == {"descriptive_leave_one_domain_out"} + ) From 961eeeb1e44d9e482b3cc76cc94df2ee816fb7d5 Mon Sep 17 00:00:00 2001 From: stevezkw <53121066+stevezkw1998@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:53:49 -0700 Subject: [PATCH 5/6] Make bootstrap evaluation resumable --- .gitignore | 2 + reports/clean_reproduction.json | 28 ++-- .../improve/baselines/lopo/missingness.csv | 8 +- .../baselines/lopo/paired_differences.csv | 92 ++++++------- .../improve/baselines/loso/missingness.csv | 8 +- .../baselines/loso/paired_differences.csv | 92 ++++++------- .../exact_peptide_free/missingness.csv | 8 +- .../exact_peptide_free/paired_differences.csv | 32 ++--- .../analysis/improve/fixed/missingness.csv | 8 +- .../improve/fixed/paired_differences.csv | 32 ++--- .../improve/length_9_10/missingness.csv | 8 +- .../length_9_10/paired_differences.csv | 32 ++--- .../improve/near_overlap_free/missingness.csv | 8 +- .../near_overlap_free/paired_differences.csv | 32 ++--- .../peptide_sensitivity/missingness.csv | 8 +- .../paired_differences.csv | 32 ++--- .../missingness.csv | 8 +- .../paired_differences.csv | 32 ++--- results/analysis/rcc/metrics.json | 128 +++++++++--------- results/analysis/rcc/paired_differences.csv | 20 +-- results/manifest.json | 42 +++--- scripts/evaluate_benchmark.py | 76 ++++++++++- tests/test_evaluator_ties.py | 18 +++ 23 files changed, 428 insertions(+), 326 deletions(-) diff --git a/.gitignore b/.gitignore index 529014d..c4b09a8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ __pycache__/ *.py[cod] .pytest_cache/ .ruff_cache/ +.bootstrap_checkpoint.json +.bootstrap_checkpoint.json.tmp .coverage htmlcov/ build/ diff --git a/reports/clean_reproduction.json b/reports/clean_reproduction.json index a5e9b7d..a67b847 100644 --- a/reports/clean_reproduction.json +++ b/reports/clean_reproduction.json @@ -1,10 +1,8 @@ { "schema_version": "1.0", - "source_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "command": "make -j4 reproduce-results", - "started_at": "2026-08-20T12:17:40Z", - "completed_at": "2026-08-20T12:29:54Z", - "elapsed_seconds": 734, + "source_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", + "command": "make -C /tmp/neorepro-release-hcLY3g -j4 reproduce-results", + "completed_at": "2026-08-21T07:53:30Z", "initial_worktree_clean": true, "returncode": 0, "runtime": { @@ -16,14 +14,20 @@ }, "verification": { "ruff": "pass", - "pytest": "27 passed", - "metric_validation": "24 checks passed at tolerance 1e-12", - "maximum_metric_absolute_error": 1.1102230246251565e-16, - "byte_stable_tracked_artifacts": true, - "post_run_changed_files_excluding_manifest": [] + "pytest": "49 passed", + "manuscript_audit": "pass; 12 citations resolved; no placeholders or forbidden overclaims", + "release_boundary": "pass; 413 tracked files; zero violations", + "metric_validation": "34 checks passed at tolerance 1e-12", + "maximum_metric_absolute_error": 2.220446049250313e-16, + "generated_outputs_promoted_from_clean_clone": true, + "post_run_findings": [ + "legacy generated CSV files used CRLF and were normalized to the writer's declared LF output", + "RCC metrics contained invocation-dependent absolute paths and were replaced by clean-clone relative paths" + ] }, "expected_manifest_change": { - "project_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "project_worktree_clean": true + "project_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", + "project_worktree_clean": false, + "reason": "the manifest is generated before the reproduction's final test step" } } diff --git a/results/analysis/improve/baselines/lopo/missingness.csv b/results/analysis/improve/baselines/lopo/missingness.csv index c1c47e3..1e9c7fc 100644 --- a/results/analysis/improve/baselines/lopo/missingness.csv +++ b/results/analysis/improve/baselines/lopo/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -HLA-only LR LOPO,predicted,17475 -HLA+peptide LR LOPO,predicted,17475 -Peptide LR LOPO,predicted,17475 +predictor,status,count +HLA-only LR LOPO,predicted,17475 +HLA+peptide LR LOPO,predicted,17475 +Peptide LR LOPO,predicted,17475 diff --git a/results/analysis/improve/baselines/lopo/paired_differences.csv b/results/analysis/improve/baselines/lopo/paired_differences.csv index 8b50859..35e705f 100644 --- a/results/analysis/improve/baselines/lopo/paired_differences.csv +++ b/results/analysis/improve/baselines/lopo/paired_differences.csv @@ -1,46 +1,46 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.562322289861119,0.07206248064073639,0.0432272336198909,0.10077865151752015 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.03802950148325792,0.010817584063849035,0.0013040512335336714,0.021318739789603722 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.3241702939840256,0.2591630393493078,0.1561631429974733,0.3661550244802481 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.4962937058813608,0.17037296078530584,0.08526341930256828,0.2613709238066563 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.19168103601308425,0.1249856306535824,0.02573337786382624,0.24086538865079202 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.13187818435593066,0.12544883588421643,0.053615353704524124,0.20699299010461514 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.05544322396267425,0.0571610770978802,0.02919128458541056,0.08672210872874804 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.07953454882823269,0.06339437108942386,0.0330266870021765,0.0974934892604656 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.046264237753205706,0.04948447393881804,0.01577981645909171,0.0867659236192426 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.04431359387664594,0.040686406123354064,0.021745338836508022,0.06092781505166672 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.044919654482706545,0.026747012183960125,0.01282387235798424,0.041761184872791436 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.04431359387664594,0.0390197394566874,0.010404583887577357,0.07079106246187 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.05915812678980218,0.05431651542029429,0.02095443039701796,0.09091761560448085 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.1213465566099074,0.08449070697256542,0.0329379099352948,0.14927716608616526 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.02957906339490109,0.018820778554169536,-0.00011042603165881617,0.04106000911749209 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.6283226185735147,0.006062151928340698,-0.02726384106308529,0.03987202282874768 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.046875818599155246,0.0019712669479517095,-0.008447824496845195,0.011326186671004641 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.45,0.13333333333333336,0.0,0.27122496147919856 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.6333333333333333,0.033333333333333326,-0.10344827586206895,0.15877976190476165 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.31666666666666665,0.0,-0.10772727272727274,0.1016949152542373 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.25012340938813493,0.0072036108520121545,-0.07287285962184244,0.08286646815910921 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.11701709766513434,-0.004412796604579883,-0.038038561847143756,0.028067246340345093 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.1459883779160683,-0.0030594579984117576,-0.041580528827991056,0.03469832763901281 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.10602336890466177,-0.010274657212638025,-0.048478099238820044,0.0274142508981307 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.083,0.0020000000000000018,-0.016949152542372885,0.020312499999999997 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.07229166666666667,-0.0006250000000000006,-0.01631555640455682,0.015873436238902344 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.09166666666666667,-0.008333333333333331,-0.04091095380029806,0.0230833333333333 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.11818160611644385,-0.004706963906347381,-0.0591545174571619,0.042768502687826025 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.21154286614806259,-0.0057056025655897735,-0.0728350385466207,0.057860254892542234 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.05441761648224497,-0.0060177745331743415,-0.032539747818009876,0.01553704952974984 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.562322289861119,0.6283226185735147,-0.0660003287123957,-0.11832890982241072,-0.013342082465851564 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.03802950148325792,0.046875818599155246,-0.008846317115897326,-0.023381279249302375,0.005429276260053588 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.3241702939840256,0.45,-0.12582970601597443,-0.2732621449104854,0.017708865829779225 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.4962937058813608,0.6333333333333333,-0.1370396274519725,-0.27083766623619915,-0.00424252670827134 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.19168103601308425,0.31666666666666665,-0.1249856306535824,-0.2528735344590615,-0.002628648721055054 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.13187818435593066,0.25012340938813493,-0.11824522503220428,-0.2111754582389387,-0.0315827093793508 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.05544322396267425,0.11701709766513434,-0.061573873702460084,-0.10753137077430959,-0.02009169118529757 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.07953454882823269,0.1459883779160683,-0.06645382908783562,-0.11709273407121512,-0.022840850217881428 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.046264237753205706,0.10602336890466177,-0.05975913115145606,-0.10841227836082305,-0.016267059525814625 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.04431359387664594,0.083,-0.03868640612335406,-0.06675304151403763,-0.010740974429773268 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.044919654482706545,0.07229166666666667,-0.027372012183960126,-0.05052328585231764,-0.005996993293902756 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.04431359387664594,0.09166666666666667,-0.04735307279002073,-0.08744525528113761,-0.011389367329959632 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.05915812678980218,0.11818160611644385,-0.05902347932664167,-0.11463921312886032,-0.010987650944993977 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.1213465566099074,0.21154286614806259,-0.09019630953815519,-0.1667874400316495,-0.020624254502470137 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.02957906339490109,0.05441761648224497,-0.024838553087343878,-0.05522741371521793,0.0014403446508246328 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.562322289861119,0.07206248064073639,0.0432272336198909,0.10077865151752015 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.03802950148325792,0.010817584063849035,0.0013040512335336714,0.021318739789603722 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.3241702939840256,0.2591630393493078,0.1561631429974733,0.3661550244802481 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.4962937058813608,0.17037296078530584,0.08526341930256828,0.2613709238066563 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.19168103601308425,0.1249856306535824,0.02573337786382624,0.24086538865079202 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.13187818435593066,0.12544883588421643,0.053615353704524124,0.20699299010461514 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.05544322396267425,0.0571610770978802,0.02919128458541056,0.08672210872874804 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.07953454882823269,0.06339437108942386,0.0330266870021765,0.0974934892604656 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.046264237753205706,0.04948447393881804,0.01577981645909171,0.0867659236192426 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.04431359387664594,0.040686406123354064,0.021745338836508022,0.06092781505166672 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.044919654482706545,0.026747012183960125,0.01282387235798424,0.041761184872791436 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.04431359387664594,0.0390197394566874,0.010404583887577357,0.07079106246187 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.05915812678980218,0.05431651542029429,0.02095443039701796,0.09091761560448085 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.1213465566099074,0.08449070697256542,0.0329379099352948,0.14927716608616526 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.02957906339490109,0.018820778554169536,-0.00011042603165881617,0.04106000911749209 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.6283226185735147,0.006062151928340698,-0.02726384106308529,0.03987202282874768 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.046875818599155246,0.0019712669479517095,-0.008447824496845195,0.011326186671004641 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.45,0.13333333333333336,0.0,0.27122496147919856 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.6333333333333333,0.033333333333333326,-0.10344827586206895,0.15877976190476165 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.31666666666666665,0.0,-0.10772727272727274,0.1016949152542373 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.25012340938813493,0.0072036108520121545,-0.07287285962184244,0.08286646815910921 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.11701709766513434,-0.004412796604579883,-0.038038561847143756,0.028067246340345093 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.1459883779160683,-0.0030594579984117576,-0.041580528827991056,0.03469832763901281 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.10602336890466177,-0.010274657212638025,-0.048478099238820044,0.0274142508981307 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.083,0.0020000000000000018,-0.016949152542372885,0.020312499999999997 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.07229166666666667,-0.0006250000000000006,-0.01631555640455682,0.015873436238902344 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.09166666666666667,-0.008333333333333331,-0.04091095380029806,0.0230833333333333 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.11818160611644385,-0.004706963906347381,-0.0591545174571619,0.042768502687826025 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.21154286614806259,-0.0057056025655897735,-0.0728350385466207,0.057860254892542234 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.05441761648224497,-0.0060177745331743415,-0.032539747818009876,0.01553704952974984 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.562322289861119,0.6283226185735147,-0.0660003287123957,-0.11832890982241072,-0.013342082465851564 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.03802950148325792,0.046875818599155246,-0.008846317115897326,-0.023381279249302375,0.005429276260053588 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.3241702939840256,0.45,-0.12582970601597443,-0.2732621449104854,0.017708865829779225 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.4962937058813608,0.6333333333333333,-0.1370396274519725,-0.27083766623619915,-0.00424252670827134 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.19168103601308425,0.31666666666666665,-0.1249856306535824,-0.2528735344590615,-0.002628648721055054 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.13187818435593066,0.25012340938813493,-0.11824522503220428,-0.2111754582389387,-0.0315827093793508 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.05544322396267425,0.11701709766513434,-0.061573873702460084,-0.10753137077430959,-0.02009169118529757 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.07953454882823269,0.1459883779160683,-0.06645382908783562,-0.11709273407121512,-0.022840850217881428 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.046264237753205706,0.10602336890466177,-0.05975913115145606,-0.10841227836082305,-0.016267059525814625 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.04431359387664594,0.083,-0.03868640612335406,-0.06675304151403763,-0.010740974429773268 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.044919654482706545,0.07229166666666667,-0.027372012183960126,-0.05052328585231764,-0.005996993293902756 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.04431359387664594,0.09166666666666667,-0.04735307279002073,-0.08744525528113761,-0.011389367329959632 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.05915812678980218,0.11818160611644385,-0.05902347932664167,-0.11463921312886032,-0.010987650944993977 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.1213465566099074,0.21154286614806259,-0.09019630953815519,-0.1667874400316495,-0.020624254502470137 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.02957906339490109,0.05441761648224497,-0.024838553087343878,-0.05522741371521793,0.0014403446508246328 diff --git a/results/analysis/improve/baselines/loso/missingness.csv b/results/analysis/improve/baselines/loso/missingness.csv index ecd3c4a..e902c54 100644 --- a/results/analysis/improve/baselines/loso/missingness.csv +++ b/results/analysis/improve/baselines/loso/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -HLA-only LR LOSO,predicted,17475 -HLA+peptide LR LOSO,predicted,17475 -Peptide LR LOSO,predicted,17475 +predictor,status,count +HLA-only LR LOSO,predicted,17475 +HLA+peptide LR LOSO,predicted,17475 +Peptide LR LOSO,predicted,17475 diff --git a/results/analysis/improve/baselines/loso/paired_differences.csv b/results/analysis/improve/baselines/loso/paired_differences.csv index 28f1c55..e13db62 100644 --- a/results/analysis/improve/baselines/loso/paired_differences.csv +++ b/results/analysis/improve/baselines/loso/paired_differences.csv @@ -1,46 +1,46 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.5459065192518,0.048947804264411254,0.020457464895754698,0.07937486042024944 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.032614680056597,0.010077252856861799,0.0031023328080865167,0.020747831469716746 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.2979086632025005,0.1687580034641662,0.07155554235306978,0.2774480293319457 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.47210862793233316,0.14455803873433354,0.039983410998719326,0.24345803450121287 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.16997512612276583,0.18002487387723415,0.07716133175732295,0.28800506073439314 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.12138679992279434,0.13662641968612688,0.06263029181255392,0.21962079097314244 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.0509966130685172,0.055910582846923976,0.02846771961923004,0.08615472760019419 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.07317468265746373,0.06008822497603074,0.03135873418626387,0.09056310898748074 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.04159745451771191,0.05832264603511714,0.025782275188904233,0.09329060029745202 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.03803862528134434,0.04112804138532232,0.020686739564114716,0.06401784999313889 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.03808248493046715,0.032750848402866184,0.015999736614523856,0.050405995011715395 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.03803862528134434,0.048628041385322326,0.0213051106695626,0.0783005935087021 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.05961413544748199,0.040723161483708394,0.01119386649032003,0.07329717629672271 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.11937446972537334,0.05250218823287424,0.007045304398280664,0.0969215226731168 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.029807067723740994,0.02513657764413305,0.0029390636108198695,0.04945687861432687 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.6188880671078999,-0.02403374359168864,-0.06339992272149965,0.015856466810683638 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.04488912649824684,-0.0021971935847880444,-0.010770780756085853,0.008134134304698228 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.45,0.016666666666666663,-0.09836065573770492,0.1273167155425218 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.5833333333333334,0.033333333333333326,-0.08932629870129873,0.15517241379310343 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.2916666666666667,0.05833333333333329,-0.03448275862068961,0.15384615384615383 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.2060379930533435,0.05197522655557771,-0.024532424987950337,0.1282996253126007 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.10329449749236344,0.003612698423077737,-0.025988054059977353,0.035371420451524464 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.13007923122688464,0.0031836764066098255,-0.031152731601539863,0.037255786595359934 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.08624215640883011,0.013677944143998935,-0.01777065475939925,0.045118767142931934 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.08633333333333333,-0.007166666666666668,-0.027499999999999997,0.015695454545454523 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.07083333333333333,0.0,-0.013890027322404365,0.01428571428571429 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.08222222222222222,0.004444444444444445,-0.023503056404556823,0.03114754098360656 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.10534099765083538,-0.005003700719644996,-0.04396245631017302,0.03553267380973812 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.19087821976360195,-0.019001561805354367,-0.08353079239282275,0.042516123849899154 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.04305045836052295,0.011893187007351091,-0.009024639249381839,0.035618348876926255 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5459065192518,0.6188880671078999,-0.0729815478560999,-0.12875313741264985,-0.015537675879811655 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.032614680056597,0.04488912649824684,-0.012274446441649843,-0.024341311717841006,-0.00250709977379717 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.2979086632025005,0.45,-0.15209133679749953,-0.28980508854756826,-0.021391401117801596 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.47210862793233316,0.5833333333333334,-0.11122470540100021,-0.2410105261552087,0.021158512901603768 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.16997512612276583,0.2916666666666667,-0.12169154054390086,-0.23840263822057875,-0.016857306346675458 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.12138679992279434,0.2060379930533435,-0.08465119313054915,-0.1682251905117228,-0.013927669926753318 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.0509966130685172,0.10329449749236344,-0.05229788442384624,-0.09277793584475871,-0.014818739588729708 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.07317468265746373,0.13007923122688464,-0.05690454856942091,-0.10314165793053343,-0.013756379356776267 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.04159745451771191,0.08624215640883011,-0.0446447018911182,-0.08615668756449882,-0.00797460488548041 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.03803862528134434,0.08633333333333333,-0.04829470805198899,-0.07810111928752385,-0.020406876903893312 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.03808248493046715,0.07083333333333333,-0.032750848402866184,-0.05311419245427756,-0.013232475823288337 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.03803862528134434,0.08222222222222222,-0.04418359694087788,-0.08141855078897617,-0.010068958967455537 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.05961413544748199,0.10534099765083538,-0.04572686220335339,-0.0945324843133182,0.0013826152737870653 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.11937446972537334,0.19087821976360195,-0.0715037500382286,-0.14653335805251105,0.0024404086896257886 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.029807067723740994,0.04305045836052295,-0.013243390636781958,-0.0384699484285053,0.009001188257215744 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.5459065192518,0.048947804264411254,0.020457464895754698,0.07937486042024944 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.032614680056597,0.010077252856861799,0.0031023328080865167,0.020747831469716746 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.2979086632025005,0.1687580034641662,0.07155554235306978,0.2774480293319457 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.47210862793233316,0.14455803873433354,0.039983410998719326,0.24345803450121287 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.16997512612276583,0.18002487387723415,0.07716133175732295,0.28800506073439314 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.12138679992279434,0.13662641968612688,0.06263029181255392,0.21962079097314244 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.0509966130685172,0.055910582846923976,0.02846771961923004,0.08615472760019419 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.07317468265746373,0.06008822497603074,0.03135873418626387,0.09056310898748074 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.04159745451771191,0.05832264603511714,0.025782275188904233,0.09329060029745202 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.03803862528134434,0.04112804138532232,0.020686739564114716,0.06401784999313889 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.03808248493046715,0.032750848402866184,0.015999736614523856,0.050405995011715395 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.03803862528134434,0.048628041385322326,0.0213051106695626,0.0783005935087021 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.05961413544748199,0.040723161483708394,0.01119386649032003,0.07329717629672271 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.11937446972537334,0.05250218823287424,0.007045304398280664,0.0969215226731168 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.029807067723740994,0.02513657764413305,0.0029390636108198695,0.04945687861432687 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.6188880671078999,-0.02403374359168864,-0.06339992272149965,0.015856466810683638 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.04488912649824684,-0.0021971935847880444,-0.010770780756085853,0.008134134304698228 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.45,0.016666666666666663,-0.09836065573770492,0.1273167155425218 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.5833333333333334,0.033333333333333326,-0.08932629870129873,0.15517241379310343 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.2916666666666667,0.05833333333333329,-0.03448275862068961,0.15384615384615383 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.2060379930533435,0.05197522655557771,-0.024532424987950337,0.1282996253126007 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.10329449749236344,0.003612698423077737,-0.025988054059977353,0.035371420451524464 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.13007923122688464,0.0031836764066098255,-0.031152731601539863,0.037255786595359934 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.08624215640883011,0.013677944143998935,-0.01777065475939925,0.045118767142931934 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.08633333333333333,-0.007166666666666668,-0.027499999999999997,0.015695454545454523 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.07083333333333333,0.0,-0.013890027322404365,0.01428571428571429 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.08222222222222222,0.004444444444444445,-0.023503056404556823,0.03114754098360656 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.10534099765083538,-0.005003700719644996,-0.04396245631017302,0.03553267380973812 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.19087821976360195,-0.019001561805354367,-0.08353079239282275,0.042516123849899154 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.04305045836052295,0.011893187007351091,-0.009024639249381839,0.035618348876926255 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5459065192518,0.6188880671078999,-0.0729815478560999,-0.12875313741264985,-0.015537675879811655 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.032614680056597,0.04488912649824684,-0.012274446441649843,-0.024341311717841006,-0.00250709977379717 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.2979086632025005,0.45,-0.15209133679749953,-0.28980508854756826,-0.021391401117801596 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.47210862793233316,0.5833333333333334,-0.11122470540100021,-0.2410105261552087,0.021158512901603768 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.16997512612276583,0.2916666666666667,-0.12169154054390086,-0.23840263822057875,-0.016857306346675458 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.12138679992279434,0.2060379930533435,-0.08465119313054915,-0.1682251905117228,-0.013927669926753318 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.0509966130685172,0.10329449749236344,-0.05229788442384624,-0.09277793584475871,-0.014818739588729708 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.07317468265746373,0.13007923122688464,-0.05690454856942091,-0.10314165793053343,-0.013756379356776267 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.04159745451771191,0.08624215640883011,-0.0446447018911182,-0.08615668756449882,-0.00797460488548041 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.03803862528134434,0.08633333333333333,-0.04829470805198899,-0.07810111928752385,-0.020406876903893312 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.03808248493046715,0.07083333333333333,-0.032750848402866184,-0.05311419245427756,-0.013232475823288337 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.03803862528134434,0.08222222222222222,-0.04418359694087788,-0.08141855078897617,-0.010068958967455537 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.05961413544748199,0.10534099765083538,-0.04572686220335339,-0.0945324843133182,0.0013826152737870653 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.11937446972537334,0.19087821976360195,-0.0715037500382286,-0.14653335805251105,0.0024404086896257886 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.029807067723740994,0.04305045836052295,-0.013243390636781958,-0.0384699484285053,0.009001188257215744 diff --git a/results/analysis/improve/exact_peptide_free/missingness.csv b/results/analysis/improve/exact_peptide_free/missingness.csv index 13b5739..8bc2a84 100644 --- a/results/analysis/improve/exact_peptide_free/missingness.csv +++ b/results/analysis/improve/exact_peptide_free/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,17440 -MHCflurry,predicted,17440 -PRIME,predicted,17440 +predictor,status,count +BigMHC,predicted,17440 +MHCflurry,predicted,17440 +PRIME,predicted,17440 diff --git a/results/analysis/improve/exact_peptide_free/paired_differences.csv b/results/analysis/improve/exact_peptide_free/paired_differences.csv index 5a68b15..f4bcd81 100644 --- a/results/analysis/improve/exact_peptide_free/paired_differences.csv +++ b/results/analysis/improve/exact_peptide_free/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,17440,464,70,0.5451536296678475,0.5962768939517047,-0.05112326428385716,-0.09208135581210583,-0.007775698279297643 -BigMHC,PRIME,immunogenicity,average_precision,17440,464,70,0.03168433497806979,0.03950436924868143,-0.007820034270611634,-0.01687301958278144,-0.0007172073562236994 -BigMHC,PRIME,immunogenicity,hitrate@10,17440,464,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 -BigMHC,PRIME,immunogenicity,hitrate@20,17440,464,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 -BigMHC,PRIME,immunogenicity,hitrate@5,17440,464,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 -BigMHC,PRIME,immunogenicity,mrr,17440,464,70,0.18012307209333414,0.212032849396242,-0.031909777302907866,-0.11836977453583092,0.057584532587463225 -BigMHC,PRIME,immunogenicity,ndcg@10,17440,464,70,0.0778599427109938,0.1119592607262639,-0.034099318015270094,-0.073745379964503,0.006438278997035955 -BigMHC,PRIME,immunogenicity,ndcg@20,17440,464,70,0.10337542318970296,0.15115428649810322,-0.04777886330840027,-0.08653657001922378,-0.008577442762943959 -BigMHC,PRIME,immunogenicity,ndcg@5,17440,464,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 -BigMHC,PRIME,immunogenicity,precision@10,17440,464,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 -BigMHC,PRIME,immunogenicity,precision@20,17440,464,70,0.04833333333333333,0.07,-0.021666666666666674,-0.038709677419354833,-0.005930957128614168 -BigMHC,PRIME,immunogenicity,precision@5,17440,464,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 -BigMHC,PRIME,immunogenicity,recall@10,17440,464,70,0.07884220626867686,0.13439881633960582,-0.05555661007092896,-0.11392996669626779,-0.0030021905986799065 -BigMHC,PRIME,immunogenicity,recall@20,17440,464,70,0.14526966526192533,0.25951379903779287,-0.11424413377586753,-0.18672131749168178,-0.0445877321686607 -BigMHC,PRIME,immunogenicity,recall@5,17440,464,70,0.03804563492063492,0.058938290023816335,-0.020892655103181418,-0.04921667171932762,0.009518724498420048 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,17440,464,70,0.5451536296678475,0.5962768939517047,-0.05112326428385716,-0.09208135581210583,-0.007775698279297643 +BigMHC,PRIME,immunogenicity,average_precision,17440,464,70,0.03168433497806979,0.03950436924868143,-0.007820034270611634,-0.01687301958278144,-0.0007172073562236994 +BigMHC,PRIME,immunogenicity,hitrate@10,17440,464,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 +BigMHC,PRIME,immunogenicity,hitrate@20,17440,464,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 +BigMHC,PRIME,immunogenicity,hitrate@5,17440,464,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 +BigMHC,PRIME,immunogenicity,mrr,17440,464,70,0.18012307209333414,0.212032849396242,-0.031909777302907866,-0.11836977453583092,0.057584532587463225 +BigMHC,PRIME,immunogenicity,ndcg@10,17440,464,70,0.0778599427109938,0.1119592607262639,-0.034099318015270094,-0.073745379964503,0.006438278997035955 +BigMHC,PRIME,immunogenicity,ndcg@20,17440,464,70,0.10337542318970296,0.15115428649810322,-0.04777886330840027,-0.08653657001922378,-0.008577442762943959 +BigMHC,PRIME,immunogenicity,ndcg@5,17440,464,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 +BigMHC,PRIME,immunogenicity,precision@10,17440,464,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 +BigMHC,PRIME,immunogenicity,precision@20,17440,464,70,0.04833333333333333,0.07,-0.021666666666666674,-0.038709677419354833,-0.005930957128614168 +BigMHC,PRIME,immunogenicity,precision@5,17440,464,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 +BigMHC,PRIME,immunogenicity,recall@10,17440,464,70,0.07884220626867686,0.13439881633960582,-0.05555661007092896,-0.11392996669626779,-0.0030021905986799065 +BigMHC,PRIME,immunogenicity,recall@20,17440,464,70,0.14526966526192533,0.25951379903779287,-0.11424413377586753,-0.18672131749168178,-0.0445877321686607 +BigMHC,PRIME,immunogenicity,recall@5,17440,464,70,0.03804563492063492,0.058938290023816335,-0.020892655103181418,-0.04921667171932762,0.009518724498420048 diff --git a/results/analysis/improve/fixed/missingness.csv b/results/analysis/improve/fixed/missingness.csv index e94710d..eece4da 100644 --- a/results/analysis/improve/fixed/missingness.csv +++ b/results/analysis/improve/fixed/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -MHCflurry,predicted,17475 -BigMHC,predicted,17475 -PRIME,predicted,17475 +predictor,status,count +MHCflurry,predicted,17475 +BigMHC,predicted,17475 +PRIME,predicted,17475 diff --git a/results/analysis/improve/fixed/paired_differences.csv b/results/analysis/improve/fixed/paired_differences.csv index ea4127e..4c7d40e 100644 --- a/results/analysis/improve/fixed/paired_differences.csv +++ b/results/analysis/improve/fixed/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,17475,465,70,0.5458288293413741,0.5969085863470571,-0.05107975700568301,-0.09216109070862642,-0.007728222469350853 -BigMHC,PRIME,immunogenicity,average_precision,17475,465,70,0.03186133541911459,0.039638725720073424,-0.007777390300958836,-0.01684074393543032,-0.0006004169764923337 -BigMHC,PRIME,immunogenicity,hitrate@10,17475,465,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 -BigMHC,PRIME,immunogenicity,hitrate@20,17475,465,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 -BigMHC,PRIME,immunogenicity,hitrate@5,17475,465,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 -BigMHC,PRIME,immunogenicity,mrr,17475,465,70,0.179799793455899,0.2118375315199005,-0.03203773806400151,-0.11850015573659703,0.05702018960577634 -BigMHC,PRIME,immunogenicity,ndcg@10,17475,465,70,0.07780339513492723,0.1119592607262639,-0.03415586559133667,-0.07374672633536174,0.006436910910518223 -BigMHC,PRIME,immunogenicity,ndcg@20,17475,465,70,0.10393761632591698,0.15170159598198435,-0.047763979656067365,-0.08653882964682626,-0.008579696667541232 -BigMHC,PRIME,immunogenicity,ndcg@5,17475,465,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 -BigMHC,PRIME,immunogenicity,precision@10,17475,465,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 -BigMHC,PRIME,immunogenicity,precision@20,17475,465,70,0.04916666666666667,0.07083333333333333,-0.02166666666666666,-0.038709677419354833,-0.005930957128614168 -BigMHC,PRIME,immunogenicity,precision@5,17475,465,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 -BigMHC,PRIME,immunogenicity,recall@10,17475,465,70,0.07884220626867686,0.13437829088804587,-0.055536084619369014,-0.1139294619720491,-0.002983728978855778 -BigMHC,PRIME,immunogenicity,recall@20,17475,465,70,0.1458033270024836,0.2600474607783511,-0.1142441337758675,-0.18672131749168178,-0.044587732168660715 -BigMHC,PRIME,immunogenicity,recall@5,17475,465,70,0.03804563492063492,0.058917764572256406,-0.02087212965162149,-0.04917484913513216,0.009519246331934279 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,17475,465,70,0.5458288293413741,0.5969085863470571,-0.05107975700568301,-0.09216109070862642,-0.007728222469350853 +BigMHC,PRIME,immunogenicity,average_precision,17475,465,70,0.03186133541911459,0.039638725720073424,-0.007777390300958836,-0.01684074393543032,-0.0006004169764923337 +BigMHC,PRIME,immunogenicity,hitrate@10,17475,465,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 +BigMHC,PRIME,immunogenicity,hitrate@20,17475,465,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 +BigMHC,PRIME,immunogenicity,hitrate@5,17475,465,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 +BigMHC,PRIME,immunogenicity,mrr,17475,465,70,0.179799793455899,0.2118375315199005,-0.03203773806400151,-0.11850015573659703,0.05702018960577634 +BigMHC,PRIME,immunogenicity,ndcg@10,17475,465,70,0.07780339513492723,0.1119592607262639,-0.03415586559133667,-0.07374672633536174,0.006436910910518223 +BigMHC,PRIME,immunogenicity,ndcg@20,17475,465,70,0.10393761632591698,0.15170159598198435,-0.047763979656067365,-0.08653882964682626,-0.008579696667541232 +BigMHC,PRIME,immunogenicity,ndcg@5,17475,465,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 +BigMHC,PRIME,immunogenicity,precision@10,17475,465,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 +BigMHC,PRIME,immunogenicity,precision@20,17475,465,70,0.04916666666666667,0.07083333333333333,-0.02166666666666666,-0.038709677419354833,-0.005930957128614168 +BigMHC,PRIME,immunogenicity,precision@5,17475,465,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 +BigMHC,PRIME,immunogenicity,recall@10,17475,465,70,0.07884220626867686,0.13437829088804587,-0.055536084619369014,-0.1139294619720491,-0.002983728978855778 +BigMHC,PRIME,immunogenicity,recall@20,17475,465,70,0.1458033270024836,0.2600474607783511,-0.1142441337758675,-0.18672131749168178,-0.044587732168660715 +BigMHC,PRIME,immunogenicity,recall@5,17475,465,70,0.03804563492063492,0.058917764572256406,-0.02087212965162149,-0.04917484913513216,0.009519246331934279 diff --git a/results/analysis/improve/length_9_10/missingness.csv b/results/analysis/improve/length_9_10/missingness.csv index 9662fff..5febff4 100644 --- a/results/analysis/improve/length_9_10/missingness.csv +++ b/results/analysis/improve/length_9_10/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,15234 -MHCflurry,predicted,15234 -PRIME,predicted,15234 +predictor,status,count +BigMHC,predicted,15234 +MHCflurry,predicted,15234 +PRIME,predicted,15234 diff --git a/results/analysis/improve/length_9_10/paired_differences.csv b/results/analysis/improve/length_9_10/paired_differences.csv index ebcb3e6..e4d4230 100644 --- a/results/analysis/improve/length_9_10/paired_differences.csv +++ b/results/analysis/improve/length_9_10/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,15234,411,70,0.5473864923206571,0.6045464625320058,-0.057159970211348776,-0.09347508009984684,-0.019669288014515205 -BigMHC,PRIME,immunogenicity,average_precision,15234,411,70,0.03273558646002369,0.04101285657052852,-0.00827727011050483,-0.017562909878109576,-0.0007460626001640635 -BigMHC,PRIME,immunogenicity,hitrate@10,15234,411,70,0.4576271186440678,0.4915254237288136,-0.03389830508474578,-0.17543859649122812,0.10719866071428542 -BigMHC,PRIME,immunogenicity,hitrate@20,15234,411,70,0.6440677966101694,0.711864406779661,-0.06779661016949157,-0.190521978021978,0.05263157894736836 -BigMHC,PRIME,immunogenicity,hitrate@5,15234,411,70,0.2542372881355932,0.3559322033898305,-0.1016949152542373,-0.2333333333333333,0.01694915254237289 -BigMHC,PRIME,immunogenicity,mrr,15234,411,70,0.19875514441408423,0.21560258035172208,-0.01684743593763785,-0.09998529680350954,0.06814529199513045 -BigMHC,PRIME,immunogenicity,ndcg@10,15234,411,70,0.08832856137958059,0.11811528728499848,-0.029786725905417893,-0.06966966753810726,0.011317563351223416 -BigMHC,PRIME,immunogenicity,ndcg@20,15234,411,70,0.11776733424221655,0.16343406460510762,-0.04566673036289107,-0.08670885312190944,-0.0058556463640528425 -BigMHC,PRIME,immunogenicity,ndcg@5,15234,411,70,0.06958795373149147,0.10196400897206478,-0.03237605524057331,-0.0772162454749475,0.011905312620938384 -BigMHC,PRIME,immunogenicity,precision@10,15234,411,70,0.0576271186440678,0.08135593220338984,-0.02372881355932204,-0.046269347705914864,-0.0034468147282291067 -BigMHC,PRIME,immunogenicity,precision@20,15234,411,70,0.05338983050847458,0.07203389830508475,-0.018644067796610167,-0.03359442349137931,-0.004310344827586202 -BigMHC,PRIME,immunogenicity,precision@5,15234,411,70,0.054237288135593226,0.10508474576271187,-0.05084745762711865,-0.08888888888888888,-0.016666666666666663 -BigMHC,PRIME,immunogenicity,recall@10,15234,411,70,0.09148901945512115,0.14732567338499541,-0.05583665392987426,-0.11633721127617748,-0.0006664995785759249 -BigMHC,PRIME,immunogenicity,recall@20,15234,411,70,0.166508324414212,0.283096470384606,-0.11658814597039399,-0.19161496564990732,-0.044268565113084475 -BigMHC,PRIME,immunogenicity,recall@5,15234,411,70,0.0489581040428498,0.06835276305615288,-0.019394659013303082,-0.053260956817491156,0.015083855449321512 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,15234,411,70,0.5473864923206571,0.6045464625320058,-0.057159970211348776,-0.09347508009984684,-0.019669288014515205 +BigMHC,PRIME,immunogenicity,average_precision,15234,411,70,0.03273558646002369,0.04101285657052852,-0.00827727011050483,-0.017562909878109576,-0.0007460626001640635 +BigMHC,PRIME,immunogenicity,hitrate@10,15234,411,70,0.4576271186440678,0.4915254237288136,-0.03389830508474578,-0.17543859649122812,0.10719866071428542 +BigMHC,PRIME,immunogenicity,hitrate@20,15234,411,70,0.6440677966101694,0.711864406779661,-0.06779661016949157,-0.190521978021978,0.05263157894736836 +BigMHC,PRIME,immunogenicity,hitrate@5,15234,411,70,0.2542372881355932,0.3559322033898305,-0.1016949152542373,-0.2333333333333333,0.01694915254237289 +BigMHC,PRIME,immunogenicity,mrr,15234,411,70,0.19875514441408423,0.21560258035172208,-0.01684743593763785,-0.09998529680350954,0.06814529199513045 +BigMHC,PRIME,immunogenicity,ndcg@10,15234,411,70,0.08832856137958059,0.11811528728499848,-0.029786725905417893,-0.06966966753810726,0.011317563351223416 +BigMHC,PRIME,immunogenicity,ndcg@20,15234,411,70,0.11776733424221655,0.16343406460510762,-0.04566673036289107,-0.08670885312190944,-0.0058556463640528425 +BigMHC,PRIME,immunogenicity,ndcg@5,15234,411,70,0.06958795373149147,0.10196400897206478,-0.03237605524057331,-0.0772162454749475,0.011905312620938384 +BigMHC,PRIME,immunogenicity,precision@10,15234,411,70,0.0576271186440678,0.08135593220338984,-0.02372881355932204,-0.046269347705914864,-0.0034468147282291067 +BigMHC,PRIME,immunogenicity,precision@20,15234,411,70,0.05338983050847458,0.07203389830508475,-0.018644067796610167,-0.03359442349137931,-0.004310344827586202 +BigMHC,PRIME,immunogenicity,precision@5,15234,411,70,0.054237288135593226,0.10508474576271187,-0.05084745762711865,-0.08888888888888888,-0.016666666666666663 +BigMHC,PRIME,immunogenicity,recall@10,15234,411,70,0.09148901945512115,0.14732567338499541,-0.05583665392987426,-0.11633721127617748,-0.0006664995785759249 +BigMHC,PRIME,immunogenicity,recall@20,15234,411,70,0.166508324414212,0.283096470384606,-0.11658814597039399,-0.19161496564990732,-0.044268565113084475 +BigMHC,PRIME,immunogenicity,recall@5,15234,411,70,0.0489581040428498,0.06835276305615288,-0.019394659013303082,-0.053260956817491156,0.015083855449321512 diff --git a/results/analysis/improve/near_overlap_free/missingness.csv b/results/analysis/improve/near_overlap_free/missingness.csv index 4a6845d..7b55edc 100644 --- a/results/analysis/improve/near_overlap_free/missingness.csv +++ b/results/analysis/improve/near_overlap_free/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,17457 -MHCflurry,predicted,17457 -PRIME,predicted,17457 +predictor,status,count +BigMHC,predicted,17457 +MHCflurry,predicted,17457 +PRIME,predicted,17457 diff --git a/results/analysis/improve/near_overlap_free/paired_differences.csv b/results/analysis/improve/near_overlap_free/paired_differences.csv index a03a4cf..7d39ed5 100644 --- a/results/analysis/improve/near_overlap_free/paired_differences.csv +++ b/results/analysis/improve/near_overlap_free/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,17457,463,70,0.5463723570585578,0.5959065466124367,-0.04953418955387889,-0.09028394644135934,-0.005571772811015889 -BigMHC,PRIME,immunogenicity,average_precision,17457,463,70,0.03180240566708497,0.03944557860693173,-0.00764317293984676,-0.016691321197734324,-0.0004495652976370504 -BigMHC,PRIME,immunogenicity,hitrate@10,17457,463,70,0.4,0.4666666666666667,-0.06666666666666665,-0.2068965517241379,0.08200136612021842 -BigMHC,PRIME,immunogenicity,hitrate@20,17457,463,70,0.5833333333333334,0.6833333333333333,-0.09999999999999998,-0.2281061259706644,0.01950757575757418 -BigMHC,PRIME,immunogenicity,hitrate@5,17457,463,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 -BigMHC,PRIME,immunogenicity,mrr,17457,463,70,0.17925826810475948,0.20999589027457913,-0.03073762216981965,-0.11675118827663264,0.058761820430309405 -BigMHC,PRIME,immunogenicity,ndcg@10,17457,463,70,0.08262114224022538,0.10875190364903926,-0.026130761408813888,-0.06586530107897698,0.016543074144428887 -BigMHC,PRIME,immunogenicity,ndcg@20,17457,463,70,0.10475573278675282,0.1484942389047597,-0.04373850611800689,-0.08338951426895236,-0.0032750982543430823 -BigMHC,PRIME,immunogenicity,ndcg@5,17457,463,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 -BigMHC,PRIME,immunogenicity,precision@10,17457,463,70,0.055,0.07833333333333334,-0.023333333333333338,-0.046434523809523814,-0.0016393442622950893 -BigMHC,PRIME,immunogenicity,precision@20,17457,463,70,0.04833333333333333,0.07,-0.021666666666666674,-0.03834482758620691,-0.0060319257744009475 -BigMHC,PRIME,immunogenicity,precision@5,17457,463,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 -BigMHC,PRIME,immunogenicity,recall@10,17457,463,70,0.09550887293534352,0.12644178295153793,-0.030932910016194415,-0.09358394741485443,0.03146490109353547 -BigMHC,PRIME,immunogenicity,recall@20,17457,463,70,0.15175570795486457,0.2521109528418432,-0.10035524488697864,-0.1788990282645159,-0.01936635838545947 -BigMHC,PRIME,immunogenicity,recall@5,17457,463,70,0.03804563492063492,0.0593145899690818,-0.021268955048446885,-0.049942376972316196,0.00936711535770567 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,17457,463,70,0.5463723570585578,0.5959065466124367,-0.04953418955387889,-0.09028394644135934,-0.005571772811015889 +BigMHC,PRIME,immunogenicity,average_precision,17457,463,70,0.03180240566708497,0.03944557860693173,-0.00764317293984676,-0.016691321197734324,-0.0004495652976370504 +BigMHC,PRIME,immunogenicity,hitrate@10,17457,463,70,0.4,0.4666666666666667,-0.06666666666666665,-0.2068965517241379,0.08200136612021842 +BigMHC,PRIME,immunogenicity,hitrate@20,17457,463,70,0.5833333333333334,0.6833333333333333,-0.09999999999999998,-0.2281061259706644,0.01950757575757418 +BigMHC,PRIME,immunogenicity,hitrate@5,17457,463,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 +BigMHC,PRIME,immunogenicity,mrr,17457,463,70,0.17925826810475948,0.20999589027457913,-0.03073762216981965,-0.11675118827663264,0.058761820430309405 +BigMHC,PRIME,immunogenicity,ndcg@10,17457,463,70,0.08262114224022538,0.10875190364903926,-0.026130761408813888,-0.06586530107897698,0.016543074144428887 +BigMHC,PRIME,immunogenicity,ndcg@20,17457,463,70,0.10475573278675282,0.1484942389047597,-0.04373850611800689,-0.08338951426895236,-0.0032750982543430823 +BigMHC,PRIME,immunogenicity,ndcg@5,17457,463,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 +BigMHC,PRIME,immunogenicity,precision@10,17457,463,70,0.055,0.07833333333333334,-0.023333333333333338,-0.046434523809523814,-0.0016393442622950893 +BigMHC,PRIME,immunogenicity,precision@20,17457,463,70,0.04833333333333333,0.07,-0.021666666666666674,-0.03834482758620691,-0.0060319257744009475 +BigMHC,PRIME,immunogenicity,precision@5,17457,463,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 +BigMHC,PRIME,immunogenicity,recall@10,17457,463,70,0.09550887293534352,0.12644178295153793,-0.030932910016194415,-0.09358394741485443,0.03146490109353547 +BigMHC,PRIME,immunogenicity,recall@20,17457,463,70,0.15175570795486457,0.2521109528418432,-0.10035524488697864,-0.1788990282645159,-0.01936635838545947 +BigMHC,PRIME,immunogenicity,recall@5,17457,463,70,0.03804563492063492,0.0593145899690818,-0.021268955048446885,-0.049942376972316196,0.00936711535770567 diff --git a/results/analysis/improve/peptide_sensitivity/missingness.csv b/results/analysis/improve/peptide_sensitivity/missingness.csv index 0c7d4f3..31e1cf8 100644 --- a/results/analysis/improve/peptide_sensitivity/missingness.csv +++ b/results/analysis/improve/peptide_sensitivity/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,15508 -MHCflurry,predicted,15508 -PRIME,predicted,15508 +predictor,status,count +BigMHC,predicted,15508 +MHCflurry,predicted,15508 +PRIME,predicted,15508 diff --git a/results/analysis/improve/peptide_sensitivity/paired_differences.csv b/results/analysis/improve/peptide_sensitivity/paired_differences.csv index ea4cb32..834b1e5 100644 --- a/results/analysis/improve/peptide_sensitivity/paired_differences.csv +++ b/results/analysis/improve/peptide_sensitivity/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5570005249823103,0.6197087694285887,-0.06270824444627832,-0.10328418049333872,-0.021200824425258242 -BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.035725091615924305,0.047085792465602884,-0.011360700849678579,-0.021756193831568165,-0.002717924156678858 -BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5333333333333333,-0.08333333333333331,-0.21666666666666667,0.05084745762711862 -BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6333333333333333,0.7666666666666667,-0.13333333333333341,-0.24137931034482762,-0.030500000000001474 -BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.25,0.38333333333333336,-0.13333333333333336,-0.27118644067796605,0.0 -BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.202555293905601,0.23299789338847923,-0.030442599482878224,-0.11356606790906844,0.058125805978220195 -BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.08917621083183522,0.13067786000916146,-0.04150164917732624,-0.08349233414665562,0.0017312696400933877 -BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.12093419366128383,0.1777591237628782,-0.05682493010159437,-0.09535555606055685,-0.016967811593794378 -BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.06847588281051294,0.11266929096539889,-0.044193408154885955,-0.09153757425491718,0.004235181157039931 -BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.06166666666666667,0.09166666666666667,-0.030000000000000006,-0.056365835777126096,-0.006666666666666668 -BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.060000000000000005,0.08416666666666667,-0.024166666666666663,-0.04051724137931034,-0.00872742200328409 -BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.05333333333333334,0.11666666666666667,-0.06333333333333332,-0.10345135467980297,-0.02413793103448276 -BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.08778671083818143,0.16338126475170087,-0.07559455391351944,-0.14399416476069116,-0.012787164224664251 -BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.16564418302755016,0.30375703286834116,-0.138112849840791,-0.21430548970982094,-0.06118383147473153 -BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.04356150793650793,0.07171790496334107,-0.02815639702683314,-0.06006275148479949,0.005094435672090141 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5570005249823103,0.6197087694285887,-0.06270824444627832,-0.10328418049333872,-0.021200824425258242 +BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.035725091615924305,0.047085792465602884,-0.011360700849678579,-0.021756193831568165,-0.002717924156678858 +BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5333333333333333,-0.08333333333333331,-0.21666666666666667,0.05084745762711862 +BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6333333333333333,0.7666666666666667,-0.13333333333333341,-0.24137931034482762,-0.030500000000001474 +BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.25,0.38333333333333336,-0.13333333333333336,-0.27118644067796605,0.0 +BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.202555293905601,0.23299789338847923,-0.030442599482878224,-0.11356606790906844,0.058125805978220195 +BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.08917621083183522,0.13067786000916146,-0.04150164917732624,-0.08349233414665562,0.0017312696400933877 +BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.12093419366128383,0.1777591237628782,-0.05682493010159437,-0.09535555606055685,-0.016967811593794378 +BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.06847588281051294,0.11266929096539889,-0.044193408154885955,-0.09153757425491718,0.004235181157039931 +BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.06166666666666667,0.09166666666666667,-0.030000000000000006,-0.056365835777126096,-0.006666666666666668 +BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.060000000000000005,0.08416666666666667,-0.024166666666666663,-0.04051724137931034,-0.00872742200328409 +BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.05333333333333334,0.11666666666666667,-0.06333333333333332,-0.10345135467980297,-0.02413793103448276 +BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.08778671083818143,0.16338126475170087,-0.07559455391351944,-0.14399416476069116,-0.012787164224664251 +BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.16564418302755016,0.30375703286834116,-0.138112849840791,-0.21430548970982094,-0.06118383147473153 +BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.04356150793650793,0.07171790496334107,-0.02815639702683314,-0.06006275148479949,0.005094435672090141 diff --git a/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv b/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv index 0c7d4f3..31e1cf8 100644 --- a/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv +++ b/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,15508 -MHCflurry,predicted,15508 -PRIME,predicted,15508 +predictor,status,count +BigMHC,predicted,15508 +MHCflurry,predicted,15508 +PRIME,predicted,15508 diff --git a/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv b/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv index c7c16fa..f4a4baa 100644 --- a/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv +++ b/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5900546578572645,0.6237143200776764,-0.033659662220411835,-0.06399324796039386,-0.004217013456002351 -BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.04431785814256973,0.0480246940858717,-0.00370683594330197,-0.0119833307880007,0.006291990015651395 -BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5666666666666667,-0.11666666666666664,-0.2459016393442623,0.014936854190585514 -BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6666666666666666,0.6666666666666666,0.0,-0.09375,0.09523809523809523 -BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.31666666666666665,0.38333333333333336,-0.06666666666666671,-0.1896756978653531,0.051724137931034475 -BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.23930852333307953,0.2560274317551143,-0.016718908422034745,-0.10748366597900222,0.07806786807751256 -BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.10539836768508622,0.1293553240052982,-0.023956956320211983,-0.06550454575548117,0.020390104084166288 -BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.14260274243018445,0.16198440048673515,-0.019381658056550705,-0.05645245831761408,0.02085912104839368 -BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.0962671022951806,0.11131066569892652,-0.015043563403745921,-0.061799664785384734,0.03481131928464634 -BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.07166666666666667,0.09,-0.018333333333333326,-0.03870967741935484,0.0017241379310344862 -BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.06583333333333334,0.07833333333333334,-0.012499999999999997,-0.025806451612903222,0.0 -BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.08,0.10666666666666667,-0.026666666666666672,-0.0655750512295082,0.012130681818181761 -BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.092826266964198,0.15148871285914897,-0.05866244589495097,-0.11852260865010635,-0.0013937041464451819 -BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.20158400709237423,0.2401696615870044,-0.03858565449463017,-0.08995684333749829,0.00712539702249144 -BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.05976333452626556,0.06611274623318233,-0.006349411706916774,-0.04051912971569411,0.027851384442322387 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5900546578572645,0.6237143200776764,-0.033659662220411835,-0.06399324796039386,-0.004217013456002351 +BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.04431785814256973,0.0480246940858717,-0.00370683594330197,-0.0119833307880007,0.006291990015651395 +BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5666666666666667,-0.11666666666666664,-0.2459016393442623,0.014936854190585514 +BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6666666666666666,0.6666666666666666,0.0,-0.09375,0.09523809523809523 +BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.31666666666666665,0.38333333333333336,-0.06666666666666671,-0.1896756978653531,0.051724137931034475 +BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.23930852333307953,0.2560274317551143,-0.016718908422034745,-0.10748366597900222,0.07806786807751256 +BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.10539836768508622,0.1293553240052982,-0.023956956320211983,-0.06550454575548117,0.020390104084166288 +BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.14260274243018445,0.16198440048673515,-0.019381658056550705,-0.05645245831761408,0.02085912104839368 +BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.0962671022951806,0.11131066569892652,-0.015043563403745921,-0.061799664785384734,0.03481131928464634 +BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.07166666666666667,0.09,-0.018333333333333326,-0.03870967741935484,0.0017241379310344862 +BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.06583333333333334,0.07833333333333334,-0.012499999999999997,-0.025806451612903222,0.0 +BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.08,0.10666666666666667,-0.026666666666666672,-0.0655750512295082,0.012130681818181761 +BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.092826266964198,0.15148871285914897,-0.05866244589495097,-0.11852260865010635,-0.0013937041464451819 +BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.20158400709237423,0.2401696615870044,-0.03858565449463017,-0.08995684333749829,0.00712539702249144 +BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.05976333452626556,0.06611274623318233,-0.006349411706916774,-0.04051912971569411,0.027851384442322387 diff --git a/results/analysis/rcc/metrics.json b/results/analysis/rcc/metrics.json index 9ca2681..dcb279c 100644 --- a/results/analysis/rcc/metrics.json +++ b/results/analysis/rcc/metrics.json @@ -1,5 +1,5 @@ { - "benchmark": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "benchmark": "data/processed/rcc_vaccine_benchmark.csv", "common_support": [ { "left": "BigMHC", @@ -254,7 +254,7 @@ }, "metadata": { "score_direction": "higher", - "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/bigmhc-v1.0.csv", + "source": "results/raw_predictions/rcc/bigmhc-v1.0.csv", "task": "immunogenicity", "version": "v1.0" }, @@ -263,7 +263,7 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.6518518518518518, - "ndcg@10": 0.6312873089172459, + "ndcg@10": 0.6312873089172458, "ndcg@20": 0.7738604771130371, "ndcg@5": 0.5325765609825637, "positive_bearing_patients": 9, @@ -342,8 +342,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.8333333333333333, - "ndcg@10": 0.8762995389962468, - "ndcg@20": 0.8762995389962468, + "ndcg@10": 0.8762995389962467, + "ndcg@20": 0.8762995389962467, "ndcg@5": 0.7590911424516827, "precision@10": 0.7, "precision@20": 0.6363636363636364, @@ -372,8 +372,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 1.0, - "ndcg@10": 0.6395647163312487, - "ndcg@20": 0.8837407779997418, + "ndcg@10": 0.6395647163312485, + "ndcg@20": 0.8837407779997417, "ndcg@5": 0.7227265726449519, "precision@10": 0.5, "precision@20": 0.5294117647058824, @@ -402,8 +402,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.5, - "ndcg@10": 0.47534276898500805, - "ndcg@20": 0.7203074868223488, + "ndcg@10": 0.47534276898500794, + "ndcg@20": 0.7203074868223487, "ndcg@5": 0.36005461457723403, "precision@10": 0.5, "precision@20": 0.5625, @@ -417,8 +417,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 1.0, - "ndcg@10": 0.7729548681688856, - "ndcg@20": 0.8573636767513857, + "ndcg@10": 0.7729548681688855, + "ndcg@20": 0.8573636767513856, "ndcg@5": 0.5531464700081437, "precision@10": 0.5, "precision@20": 0.5454545454545454, @@ -432,8 +432,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.41666666666666663, - "ndcg@10": 0.4628984070136053, - "ndcg@20": 0.6780957041773529, + "ndcg@10": 0.46289840701360524, + "ndcg@20": 0.6780957041773528, "ndcg@5": 0.46905661104071605, "precision@10": 0.4, "precision@20": 0.4666666666666667, @@ -684,7 +684,7 @@ }, "metadata": { "score_direction": "higher", - "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "source": "results/raw_predictions/rcc/deephlapan-1.1.1.csv", "task": "immunogenicity", "version": "1.1.1@ac1f4beb" }, @@ -693,7 +693,7 @@ "hitrate@20": 1.0, "hitrate@5": 0.8888888888888888, "mrr": 0.7222222222222222, - "ndcg@10": 0.6703258245072995, + "ndcg@10": 0.6703258245072994, "ndcg@20": 0.8085230459234908, "ndcg@5": 0.6140248752438171, "positive_bearing_patients": 9, @@ -735,7 +735,7 @@ }, "ndcg@20": { "high": 0.8947689844911606, - "low": 0.6899714059669765 + "low": 0.6899714059669764 }, "ndcg@5": { "high": 0.7654367605421801, @@ -772,8 +772,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.8333333333333333, - "ndcg@10": 0.7730836500855317, - "ndcg@20": 0.8497584640784246, + "ndcg@10": 0.7730836500855316, + "ndcg@20": 0.8497584640784245, "ndcg@5": 0.6130227926089763, "precision@10": 0.6, "precision@20": 0.6363636363636364, @@ -802,8 +802,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 1.0, - "ndcg@10": 0.6592243328151071, - "ndcg@20": 0.8935911984826842, + "ndcg@10": 0.659224332815107, + "ndcg@20": 0.893591198482684, "ndcg@5": 0.830419897363192, "precision@10": 0.5, "precision@20": 0.5294117647058824, @@ -832,8 +832,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.5, - "ndcg@10": 0.5161528906822678, - "ndcg@20": 0.7543146979573652, + "ndcg@10": 0.5161528906822677, + "ndcg@20": 0.754314697957365, "ndcg@5": 0.5296347172140421, "precision@10": 0.5, "precision@20": 0.5625, @@ -847,8 +847,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 1.0, - "ndcg@10": 0.9448192267559941, - "ndcg@20": 0.9448192267559941, + "ndcg@10": 0.944819226755994, + "ndcg@20": 0.944819226755994, "ndcg@5": 0.8539316501572937, "precision@10": 0.6, "precision@20": 0.5454545454545454, @@ -862,8 +862,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.5, - "ndcg@10": 0.5798612725328838, - "ndcg@20": 0.7275358570132512, + "ndcg@10": 0.5798612725328837, + "ndcg@20": 0.7275358570132511, "ndcg@5": 0.38356636737133565, "precision@10": 0.5, "precision@20": 0.4666666666666667, @@ -1009,7 +1009,7 @@ }, "metadata": { "score_direction": "higher", - "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deepimmuno-cnn.csv", + "source": "results/raw_predictions/rcc/deepimmuno-cnn.csv", "task": "immunogenicity", "version": "1.0@df42ac5b" }, @@ -1158,7 +1158,7 @@ "hitrate@5": 1.0, "mrr": 0.5, "ndcg@10": 0.6108096776415199, - "ndcg@20": 0.7347887996819422, + "ndcg@20": 0.7347887996819421, "ndcg@5": 0.5147714448836774, "precision@10": 0.45, "precision@20": 0.5, @@ -1409,7 +1409,7 @@ }, "metadata": { "score_direction": "higher", - "source": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/prime-2.0.csv", + "source": "results/raw_predictions/rcc/prime-2.0.csv", "task": "immunogenicity", "version": "2.0" }, @@ -1419,7 +1419,7 @@ "hitrate@5": 1.0, "mrr": 0.7685185185185185, "ndcg@10": 0.6881137179921539, - "ndcg@20": 0.8271212105514111, + "ndcg@20": 0.827121210551411, "ndcg@5": 0.6908650160295152, "positive_bearing_patients": 9, "precision@10": 0.5611111111111111, @@ -1456,11 +1456,11 @@ }, "ndcg@10": { "high": 0.7462504477591396, - "low": 0.6222563694605704 + "low": 0.6222563694605703 }, "ndcg@20": { - "high": 0.884401594746618, - "low": 0.7626559123915736 + "high": 0.8844015947466178, + "low": 0.7626559123915735 }, "ndcg@5": { "high": 0.789022108580068, @@ -1497,8 +1497,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.8333333333333333, - "ndcg@10": 0.8047526145467748, - "ndcg@20": 0.8814274285396677, + "ndcg@10": 0.8047526145467747, + "ndcg@20": 0.8814274285396676, "ndcg@5": 0.7590911424516827, "precision@10": 0.6, "precision@20": 0.6363636363636364, @@ -1527,8 +1527,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.5, - "ndcg@10": 0.5030399532861565, - "ndcg@20": 0.7513611520351536, + "ndcg@10": 0.5030399532861564, + "ndcg@20": 0.7513611520351534, "ndcg@5": 0.5147714448836774, "precision@10": 0.5, "precision@20": 0.5294117647058824, @@ -1557,8 +1557,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.5, - "ndcg@10": 0.6866059341140196, - "ndcg@20": 0.8042717497692193, + "ndcg@10": 0.6866059341140195, + "ndcg@20": 0.8042717497692192, "ndcg@5": 0.6608397947263839, "precision@10": 0.7, "precision@20": 0.5625, @@ -1572,8 +1572,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 1.0, - "ndcg@10": 0.7891499901405781, - "ndcg@20": 0.8735587987230782, + "ndcg@10": 0.789149990140578, + "ndcg@20": 0.8735587987230781, "ndcg@5": 0.6843515475204855, "precision@10": 0.5, "precision@20": 0.5454545454545454, @@ -1587,8 +1587,8 @@ "hitrate@20": 1.0, "hitrate@5": 1.0, "mrr": 0.75, - "ndcg@10": 0.6573429591465089, - "ndcg@20": 0.8003441152775891, + "ndcg@10": 0.6573429591465088, + "ndcg@20": 0.800344115277589, "ndcg@5": 0.5922216874835863, "precision@10": 0.5, "precision@20": 0.4666666666666667, @@ -1733,20 +1733,20 @@ }, { "ci_high": 0.012336267625158463, - "ci_low": -0.08993220825260795, + "ci_low": -0.08993220825260806, "difference_left_minus_right": -0.03903851559005356, "left": "BigMHC", - "left_value_common": 0.6312873089172459, + "left_value_common": 0.6312873089172458, "metric": "ndcg@10", "n_common": 128, "patients_common": 9, "positives_common": 75, "right": "DeepHLApan", - "right_value_common": 0.6703258245072995, + "right_value_common": 0.6703258245072994, "task": "immunogenicity" }, { - "ci_high": -0.009628459310849956, + "ci_high": -0.009628459310849847, "ci_low": -0.06086790188393626, "difference_left_minus_right": -0.03466256881045371, "left": "BigMHC", @@ -1956,7 +1956,7 @@ "task": "immunogenicity" }, { - "ci_high": 0.040171356613644446, + "ci_high": 0.040171356613644335, "ci_low": -0.0973475628913677, "difference_left_minus_right": -0.01610746654958295, "left": "BigMHC", @@ -2153,10 +2153,10 @@ }, { "ci_high": 0.03017239737900372, - "ci_low": -0.1448315833102618, - "difference_left_minus_right": -0.05682640907490799, + "ci_low": -0.14483158331026183, + "difference_left_minus_right": -0.0568264090749081, "left": "BigMHC", - "left_value_common": 0.6312873089172459, + "left_value_common": 0.6312873089172458, "metric": "ndcg@10", "n_common": 128, "patients_common": 9, @@ -2166,9 +2166,9 @@ "task": "immunogenicity" }, { - "ci_high": 0.008075273494051982, - "ci_low": -0.11794338501591044, - "difference_left_minus_right": -0.05326073343837401, + "ci_high": 0.008075273494052092, + "ci_low": -0.11794338501591053, + "difference_left_minus_right": -0.0532607334383739, "left": "BigMHC", "left_value_common": 0.7738604771130371, "metric": "ndcg@20", @@ -2176,7 +2176,7 @@ "patients_common": 9, "positives_common": 75, "right": "PRIME", - "right_value_common": 0.8271212105514111, + "right_value_common": 0.827121210551411, "task": "immunogenicity" }, { @@ -2362,7 +2362,7 @@ "task": "immunogenicity" }, { - "ci_high": 0.12901205374691374, + "ci_high": 0.12901205374691363, "ci_low": -0.04765365130313093, "difference_left_minus_right": 0.03239542824625208, "left": "DeepHLApan", @@ -2376,7 +2376,7 @@ "task": "immunogenicity" }, { - "ci_high": 0.0819166098642451, + "ci_high": 0.08191660986424498, "ci_low": -0.05020882164937479, "difference_left_minus_right": 0.014684125097620404, "left": "DeepHLApan", @@ -2572,11 +2572,11 @@ "task": "immunogenicity" }, { - "ci_high": 0.08331803966539883, + "ci_high": 0.08331803966539872, "ci_low": -0.12058836256824594, - "difference_left_minus_right": -0.017787893484854433, + "difference_left_minus_right": -0.017787893484854544, "left": "DeepHLApan", - "left_value_common": 0.6703258245072995, + "left_value_common": 0.6703258245072994, "metric": "ndcg@10", "n_common": 128, "patients_common": 9, @@ -2587,8 +2587,8 @@ }, { "ci_high": 0.04597761031229554, - "ci_low": -0.09102404902752281, - "difference_left_minus_right": -0.0185981646279203, + "ci_low": -0.0910240490275227, + "difference_left_minus_right": -0.01859816462792019, "left": "DeepHLApan", "left_value_common": 0.8085230459234908, "metric": "ndcg@20", @@ -2596,7 +2596,7 @@ "patients_common": 9, "positives_common": 75, "right": "PRIME", - "right_value_common": 0.8271212105514111, + "right_value_common": 0.827121210551411, "task": "immunogenicity" }, { @@ -2797,7 +2797,7 @@ }, { "ci_high": 0.05207977361323347, - "ci_low": -0.03209450981710734, + "ci_low": -0.03209450981710745, "difference_left_minus_right": 0.009740393072505737, "left": "DeepImmuno-CNN", "left_value_common": 0.8368671618581434, diff --git a/results/analysis/rcc/paired_differences.csv b/results/analysis/rcc/paired_differences.csv index b8c0ecf..255dbda 100644 --- a/results/analysis/rcc/paired_differences.csv +++ b/results/analysis/rcc/paired_differences.csv @@ -5,8 +5,8 @@ BigMHC,DeepHLApan,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 BigMHC,DeepHLApan,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 BigMHC,DeepHLApan,immunogenicity,hitrate@5,128,75,9,1.0,0.8888888888888888,0.11111111111111116,0.0,0.33333333333333337 BigMHC,DeepHLApan,immunogenicity,mrr,128,75,9,0.6518518518518518,0.7222222222222222,-0.07037037037037042,-0.18148148148148147,0.001898148148147924 -BigMHC,DeepHLApan,immunogenicity,ndcg@10,128,75,9,0.6312873089172459,0.6703258245072995,-0.03903851559005356,-0.08993220825260795,0.012336267625158463 -BigMHC,DeepHLApan,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.8085230459234908,-0.03466256881045371,-0.06086790188393626,-0.009628459310849956 +BigMHC,DeepHLApan,immunogenicity,ndcg@10,128,75,9,0.6312873089172458,0.6703258245072994,-0.03903851559005356,-0.08993220825260806,0.012336267625158463 +BigMHC,DeepHLApan,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.8085230459234908,-0.03466256881045371,-0.06086790188393626,-0.009628459310849847 BigMHC,DeepHLApan,immunogenicity,ndcg@5,128,75,9,0.5325765609825637,0.6140248752438171,-0.08144831426125343,-0.1901851493238087,0.050409043936959864 BigMHC,DeepHLApan,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.0444444444444444,0.0444444444444444 BigMHC,DeepHLApan,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 @@ -21,7 +21,7 @@ BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 BigMHC,DeepImmuno-CNN,immunogenicity,mrr,51,29,9,0.7619047619047619,0.8095238095238095,-0.04761904761904767,-0.3214285714285714,0.125 BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@10,51,29,9,0.8087012940824889,0.8191558587095117,-0.010454564627022767,-0.09325707120581819,0.05335537459609574 -BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8207596953085604,0.8368671618581434,-0.01610746654958295,-0.0973475628913677,0.040171356613644446 +BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8207596953085604,0.8368671618581434,-0.01610746654958295,-0.0973475628913677,0.040171356613644335 BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@5,51,29,9,0.6969262980316332,0.7107525131913099,-0.013826215159676747,-0.09740725277182755,0.044278942373923366 BigMHC,DeepImmuno-CNN,immunogenicity,precision@10,51,29,9,0.6232993197278911,0.616156462585034,0.0071428571428571175,0.0,0.022222222222222143 BigMHC,DeepImmuno-CNN,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 @@ -35,8 +35,8 @@ BigMHC,PRIME,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 BigMHC,PRIME,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 BigMHC,PRIME,immunogenicity,hitrate@5,128,75,9,1.0,1.0,0.0,0.0,0.0 BigMHC,PRIME,immunogenicity,mrr,128,75,9,0.6518518518518518,0.7685185185185185,-0.1166666666666667,-0.30370370370370364,0.07407407407407407 -BigMHC,PRIME,immunogenicity,ndcg@10,128,75,9,0.6312873089172459,0.6881137179921539,-0.05682640907490799,-0.1448315833102618,0.03017239737900372 -BigMHC,PRIME,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.8271212105514111,-0.05326073343837401,-0.11794338501591044,0.008075273494051982 +BigMHC,PRIME,immunogenicity,ndcg@10,128,75,9,0.6312873089172458,0.6881137179921539,-0.0568264090749081,-0.14483158331026183,0.03017239737900372 +BigMHC,PRIME,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.827121210551411,-0.0532607334383739,-0.11794338501591053,0.008075273494052092 BigMHC,PRIME,immunogenicity,ndcg@5,128,75,9,0.5325765609825637,0.6908650160295152,-0.15828845504695155,-0.2990950049162747,-0.023271734532436354 BigMHC,PRIME,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.06666666666666665,0.06666666666666676 BigMHC,PRIME,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 @@ -50,8 +50,8 @@ DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@10,51,29,9,1.0,1.0,0.0,0.0,0.0 DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 DeepHLApan,DeepImmuno-CNN,immunogenicity,mrr,51,29,9,0.8452380952380952,0.8095238095238095,0.0357142857142857,-0.09999999999999998,0.20020833333333227 -DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@10,51,29,9,0.8515512869557638,0.8191558587095117,0.03239542824625208,-0.04765365130313093,0.12901205374691374 -DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8515512869557638,0.8368671618581434,0.014684125097620404,-0.05020882164937479,0.0819166098642451 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@10,51,29,9,0.8515512869557638,0.8191558587095117,0.03239542824625208,-0.04765365130313093,0.12901205374691363 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8515512869557638,0.8368671618581434,0.014684125097620404,-0.05020882164937479,0.08191660986424498 DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@5,51,29,9,0.7264811600913925,0.7107525131913099,0.015728646900082577,-0.0833180287588855,0.11382296763686184 DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@10,51,29,9,0.6375850340136054,0.616156462585034,0.021428571428571463,0.0,0.06428571428571428 DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 @@ -65,8 +65,8 @@ DeepHLApan,PRIME,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 DeepHLApan,PRIME,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 DeepHLApan,PRIME,immunogenicity,hitrate@5,128,75,9,0.8888888888888888,1.0,-0.11111111111111116,-0.33333333333333337,0.0 DeepHLApan,PRIME,immunogenicity,mrr,128,75,9,0.7222222222222222,0.7685185185185185,-0.04629629629629628,-0.2314814814814815,0.12962962962962965 -DeepHLApan,PRIME,immunogenicity,ndcg@10,128,75,9,0.6703258245072995,0.6881137179921539,-0.017787893484854433,-0.12058836256824594,0.08331803966539883 -DeepHLApan,PRIME,immunogenicity,ndcg@20,128,75,9,0.8085230459234908,0.8271212105514111,-0.0185981646279203,-0.09102404902752281,0.04597761031229554 +DeepHLApan,PRIME,immunogenicity,ndcg@10,128,75,9,0.6703258245072994,0.6881137179921539,-0.017787893484854544,-0.12058836256824594,0.08331803966539872 +DeepHLApan,PRIME,immunogenicity,ndcg@20,128,75,9,0.8085230459234908,0.827121210551411,-0.01859816462792019,-0.0910240490275227,0.04597761031229554 DeepHLApan,PRIME,immunogenicity,ndcg@5,128,75,9,0.6140248752438171,0.6908650160295152,-0.07684014078569812,-0.26593716874969686,0.09313139286054935 DeepHLApan,PRIME,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.06666666666666665,0.06666666666666676 DeepHLApan,PRIME,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 @@ -81,7 +81,7 @@ DeepImmuno-CNN,PRIME,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 DeepImmuno-CNN,PRIME,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 DeepImmuno-CNN,PRIME,immunogenicity,mrr,51,29,9,0.8095238095238095,0.7738095238095238,0.0357142857142857,-0.09999999999999998,0.20000000000000007 DeepImmuno-CNN,PRIME,immunogenicity,ndcg@10,51,29,9,0.8191558587095117,0.8271267687856376,-0.007970910076125937,-0.0835917238927503,0.05146739445932063 -DeepImmuno-CNN,PRIME,immunogenicity,ndcg@20,51,29,9,0.8368671618581434,0.8271267687856376,0.009740393072505737,-0.03209450981710734,0.05207977361323347 +DeepImmuno-CNN,PRIME,immunogenicity,ndcg@20,51,29,9,0.8368671618581434,0.8271267687856376,0.009740393072505737,-0.03209450981710745,0.05207977361323347 DeepImmuno-CNN,PRIME,immunogenicity,ndcg@5,51,29,9,0.7107525131913099,0.7070896355777788,0.0036628776135311236,-0.08016590563417962,0.0826002139424956 DeepImmuno-CNN,PRIME,immunogenicity,precision@10,51,29,9,0.616156462585034,0.6375850340136054,-0.021428571428571463,-0.06428571428571428,0.0 DeepImmuno-CNN,PRIME,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 diff --git a/results/manifest.json b/results/manifest.json index a03be78..58a36a6 100644 --- a/results/manifest.json +++ b/results/manifest.json @@ -176,9 +176,9 @@ "sha256": "54dc400f5e5970188b4822ae9253e17c3e2b18a463056a2e629f7f884e44589d" }, { - "bytes": 889, + "bytes": 1314, "path": "reports/clean_reproduction.json", - "sha256": "7ff012366417a31a5136f85e7a3be6bcd9135a77545078d26938abb699b3c02d" + "sha256": "a5ec65a24bafd59a03d383d261c45f95cfaeea74c69770603307435aab96897b" }, { "bytes": 3957, @@ -426,9 +426,9 @@ "sha256": "c5efc343344eff9f5c962f8d7834c7f3c0c686bcc669eaaf87617b44f0da3b00" }, { - "bytes": 83351, + "bytes": 82938, "path": "results/analysis/rcc/metrics.json", - "sha256": "25ebbf73dd025d75b9d94f646022dfa4eaa794a3f80f47d81478cbe93df1fe60" + "sha256": "6da6a361e1f7bf5596e88820cb2c2c00d549e3152579f30e217da446e2daf859" }, { "bytes": 266, @@ -436,9 +436,9 @@ "sha256": "75b876d69948ea61012702b39a0382bed0a4b41ed40890c4985a0827a14a1d33" }, { - "bytes": 11452, + "bytes": 11450, "path": "results/analysis/rcc/paired_differences.csv", - "sha256": "97e77b6a54cd871b34e9531cda9ef8e321fd6f2481387379d04c40475c365be8" + "sha256": "cc67f00b4b278f0614bcac1b2b4ef9513ecc6abbb144bce1d44696e309885449" }, { "bytes": 1402, @@ -554,12 +554,12 @@ "studies": 3 }, "clean_reproduction": { - "command": "make -j4 reproduce-results", - "completed_at": "2026-08-20T12:29:54Z", - "elapsed_seconds": 734, + "command": "make -C /tmp/neorepro-release-hcLY3g -j4 reproduce-results", + "completed_at": "2026-08-21T07:53:30Z", "expected_manifest_change": { - "project_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "project_worktree_clean": true + "project_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", + "project_worktree_clean": false, + "reason": "the manifest is generated before the reproduction's final test step" }, "initial_worktree_clean": true, "returncode": 0, @@ -571,14 +571,18 @@ "scipy": "1.17.1" }, "schema_version": "1.0", - "source_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "started_at": "2026-08-20T12:17:40Z", + "source_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", "verification": { - "byte_stable_tracked_artifacts": true, - "maximum_metric_absolute_error": 1.1102230246251565e-16, - "metric_validation": "24 checks passed at tolerance 1e-12", - "post_run_changed_files_excluding_manifest": [], - "pytest": "27 passed", + "generated_outputs_promoted_from_clean_clone": true, + "manuscript_audit": "pass; 12 citations resolved; no placeholders or forbidden overclaims", + "maximum_metric_absolute_error": 2.220446049250313e-16, + "metric_validation": "34 checks passed at tolerance 1e-12", + "post_run_findings": [ + "legacy generated CSV files used CRLF and were normalized to the writer's declared LF output", + "RCC metrics contained invocation-dependent absolute paths and were replaced by clean-clone relative paths" + ], + "pytest": "49 passed", + "release_boundary": "pass; 413 tracked files; zero violations", "ruff": "pass" } }, @@ -609,7 +613,7 @@ "MHCflurry": "2.2.1", "PRIME": "2.0" }, - "project_commit": "db00e769f8d6831221264e6a187309cfeadbd91a", + "project_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", "project_worktree_clean": false, "schema_version": "1.0" } diff --git a/scripts/evaluate_benchmark.py b/scripts/evaluate_benchmark.py index 2abf6a6..c0ab2ae 100755 --- a/scripts/evaluate_benchmark.py +++ b/scripts/evaluate_benchmark.py @@ -4,7 +4,9 @@ from __future__ import annotations import argparse +import ast import csv +import hashlib import json import math import random @@ -181,6 +183,26 @@ def load_csv(path: Path) -> list[dict[str, str]]: return list(csv.DictReader(handle)) +def input_fingerprint(paths: list[Path], bootstrap: int, seed: int) -> str: + """Bind a resumable bootstrap checkpoint to its exact inputs and configuration.""" + digest = hashlib.sha256() + digest.update(f"bootstrap={bootstrap}\nseed={seed}\n".encode()) + for path in paths: + digest.update(str(path).encode()) + digest.update(b"\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, sort_keys=True, allow_nan=False) + "\n") + temporary.replace(path) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--benchmark", type=Path, required=True) @@ -188,6 +210,12 @@ def main() -> int: parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--bootstrap", type=int, default=2000) parser.add_argument("--seed", type=int, default=20260820) + parser.add_argument( + "--checkpoint-every", + type=int, + default=50, + help="atomically save bootstrap state every N replicates; 0 disables resume", + ) args = parser.parse_args() benchmark_rows = load_csv(args.benchmark) @@ -333,7 +361,32 @@ def main() -> int: } for predictor, rows in joined.items() } - for _ in range(args.bootstrap): + checkpoint_path = args.output_dir / ".bootstrap_checkpoint.json" + fingerprint = input_fingerprint( + [args.benchmark, *sorted(args.predictions)], args.bootstrap, args.seed + ) + start_replicate = 0 + if args.checkpoint_every > 0 and checkpoint_path.exists(): + checkpoint = json.loads(checkpoint_path.read_text()) + if checkpoint.get("fingerprint") != fingerprint: + raise SystemExit( + f"checkpoint inputs/configuration changed; remove {checkpoint_path} to restart" + ) + start_replicate = int(checkpoint["completed"]) + rng.setstate(ast.literal_eval(checkpoint["rng_state"])) + samples = { + predictor: defaultdict(list, metric_samples) + for predictor, metric_samples in checkpoint["samples"].items() + } + restored_pairs = {} + for item in checkpoint["paired_samples"]: + restored_pairs[(item["left"], item["right"])] = defaultdict( + list, item["metrics"] + ) + paired_samples = restored_pairs + print(f"resuming bootstrap at replicate {start_replicate}/{args.bootstrap}") + + for replicate_index in range(start_replicate, args.bootstrap): draws = rng.choices(patients, k=len(patients)) replicate = {} for predictor in joined: @@ -383,6 +436,26 @@ def main() -> int: right_value = pair_replicate[right][metric] if isinstance(left_value, (int, float)) and isinstance(right_value, (int, float)): paired_samples[pair][metric].append(left_value - right_value) + completed = replicate_index + 1 + if ( + args.checkpoint_every > 0 + and completed < args.bootstrap + and completed % args.checkpoint_every == 0 + ): + atomic_json( + checkpoint_path, + { + "schema_version": 1, + "fingerprint": fingerprint, + "completed": completed, + "rng_state": repr(rng.getstate()), + "samples": samples, + "paired_samples": [ + {"left": pair[0], "right": pair[1], "metrics": values} + for pair, values in sorted(paired_samples.items()) + ], + }, + ) for predictor, predictor_metrics in metrics.items(): predictor_metrics["patient_bootstrap_95ci"] = { @@ -449,6 +522,7 @@ def main() -> int: writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") writer.writeheader() writer.writerows(paired) + checkpoint_path.unlink(missing_ok=True) print( json.dumps( { diff --git a/tests/test_evaluator_ties.py b/tests/test_evaluator_ties.py index 30483b5..35e3700 100644 --- a/tests/test_evaluator_ties.py +++ b/tests/test_evaluator_ties.py @@ -1,5 +1,7 @@ import importlib.util +import json import math +import random from pathlib import Path import pytest @@ -30,3 +32,19 @@ def test_patient_top_k_uses_analytic_expectation_for_ties() -> None: assert values["mrr"] == pytest.approx(sum(1 / rank for rank in range(1, 11)) / 10) expected_dcg = 0.1 * sum(1 / math.log2(rank + 1) for rank in range(1, 6)) assert values["ndcg@5"] == pytest.approx(expected_dcg) + + +def test_bootstrap_checkpoint_helpers_are_atomic_and_input_bound(tmp_path: Path) -> None: + source = tmp_path / "input.csv" + source.write_text("a,b\n1,2\n") + first = MODULE.input_fingerprint([source], 2000, 7) + assert first == MODULE.input_fingerprint([source], 2000, 7) + assert first != MODULE.input_fingerprint([source], 2000, 8) + source.write_text("a,b\n1,3\n") + assert first != MODULE.input_fingerprint([source], 2000, 7) + + checkpoint = tmp_path / ".bootstrap_checkpoint.json" + state = {"completed": 50, "rng_state": repr(random.Random(7).getstate())} + MODULE.atomic_json(checkpoint, state) + assert json.loads(checkpoint.read_text()) == state + assert not checkpoint.with_suffix(".json.tmp").exists() From d8710676d758a51701121c6ab04c32183c8ac553 Mon Sep 17 00:00:00 2001 From: stevezkw <53121066+stevezkw1998@users.noreply.github.com> Date: Fri, 21 Aug 2026 01:39:19 -0700 Subject: [PATCH 6/6] Add portable standard benchmark entry point --- README.de.md | 1 + README.es.md | 1 + README.fr.md | 1 + README.ja.md | 1 + README.ko.md | 1 + README.md | 1 + README.zh-CN.md | 1 + README.zh-TW.md | 1 + contracts/README.md | 33 +- contracts/synthetic/README.md | 7 + contracts/synthetic/standard_predictions.csv | 9 + i18n/readme_state.json | 16 +- src/neorepro/benchmark.py | 615 +++++++++++++++++++ src/neorepro/cli.py | 49 ++ tests/test_standard_benchmark.py | 74 +++ 15 files changed, 802 insertions(+), 9 deletions(-) create mode 100644 contracts/synthetic/standard_predictions.csv create mode 100644 src/neorepro/benchmark.py create mode 100644 tests/test_standard_benchmark.py diff --git a/README.de.md b/README.de.md index e0c00a0..42151a2 100644 --- a/README.de.md +++ b/README.de.md @@ -15,6 +15,7 @@ Der Beitrag ist eine Benchmark- und Forschungsressource, kein neuer Prädiktor u - **Aktuelles Manuskript:** [als Ressource positioniertes Manuskript](paper/manuscript_resource.md). - **Evidenz in zwei Minuten:** [zweisprachige Expertenübersicht](output/pdf/neorepro_expert_brief_bilingual.pdf) und [Zusammenfassung der unabhängigen Kohortenerweiterung](reports/extension_summary.md). - **Festgeschriebene Ergebnisse reproduzieren:** den Befehl im Abschnitt „Reproduktion“ verwenden. +- **Eigenen Prädiktor benchmarken:** `neorepro benchmark predictions.csv --output-dir neorepro-results` ausführen; siehe [Standardvertrag für eine einzelne Datei](contracts/README.md). - **Drittanbieter-Datensatz oder -Prädiktor hinzufügen:** den [plug-in contract](contracts/README.md) verwenden. - **Feste Version zitieren:** [CITATION.cff](CITATION.cff) und [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). diff --git a/README.es.md b/README.es.md index 604ecf2..0806ba0 100644 --- a/README.es.md +++ b/README.es.md @@ -15,6 +15,7 @@ Es una contribución de referencia y recursos, no un predictor nuevo ni una afir - **Manuscrito actual:** [manuscrito orientado como recurso](paper/manuscript_resource.md). - **Resumen de la evidencia en dos minutos:** [informe bilingüe para expertos](output/pdf/neorepro_expert_brief_bilingual.pdf) y [resumen de la extensión con cohorte independiente](reports/extension_summary.md). - **Reproducir los resultados fijados:** usar el comando de la sección «Reproducción». +- **Evaluar su propio predictor:** ejecutar `neorepro benchmark predictions.csv --output-dir neorepro-results`; consultar el [contrato estándar de un solo archivo](contracts/README.md). - **Añadir un conjunto de datos o predictor de terceros:** consultar el [plug-in contract](contracts/README.md). - **Citar una versión fija:** [CITATION.cff](CITATION.cff) y [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). diff --git a/README.fr.md b/README.fr.md index 87de1c9..e59f866 100644 --- a/README.fr.md +++ b/README.fr.md @@ -15,6 +15,7 @@ Il s’agit d’une contribution de type ressource et benchmark, et non d’un n - **Manuscrit actuel :** [manuscrit positionné comme ressource](paper/manuscript_resource.md). - **Synthèse des preuves en deux minutes :** [note bilingue pour experts](output/pdf/neorepro_expert_brief_bilingual.pdf) et [résumé de l’extension sur une cohorte indépendante](reports/extension_summary.md). - **Reproduire les résultats figés :** utiliser la commande de la section « Reproduction » ci-dessous. +- **Évaluer votre propre prédicteur :** exécuter `neorepro benchmark predictions.csv --output-dir neorepro-results` ; consulter le [contrat standard à fichier unique](contracts/README.md). - **Ajouter un jeu de données ou un prédicteur tiers :** consulter le [plug-in contract](contracts/README.md). - **Citer une version figée :** [CITATION.cff](CITATION.cff) et [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). diff --git a/README.ja.md b/README.ja.md index d8835d0..92d0b67 100644 --- a/README.ja.md +++ b/README.ja.md @@ -15,6 +15,7 @@ NeoRepro は、公開されている MHC-I ペプチド–HLA ネオアンチゲ - **現行原稿：**[リソースとして位置付けた原稿](paper/manuscript_resource.md)。 - **短時間で読めるエビデンス概要：**[中英併記の専門家向け概要](output/pdf/neorepro_expert_brief_bilingual.pdf)および[独立コホート拡張の概要](reports/extension_summary.md)。 - **凍結済み結果の再現：**下記「再現」節のコマンドを使用してください。 +- **独自予測器のベンチマーク：**`neorepro benchmark predictions.csv --output-dir neorepro-results` を実行し、[標準単一ファイル契約](contracts/README.md)を参照してください。 - **第三者データセットまたは予測器の追加：**[plug-in contract](contracts/README.md) を参照してください。 - **固定版の引用：**[CITATION.cff](CITATION.cff) および [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)を参照してください。 diff --git a/README.ko.md b/README.ko.md index 40df0b7..a9ad31a 100644 --- a/README.ko.md +++ b/README.ko.md @@ -15,6 +15,7 @@ NeoRepro는 공개 MHC-I 펩타이드–HLA 신생항원 예측기를 위한 데 - **현재 원고:** [리소스 중심 원고](paper/manuscript_resource.md). - **간단한 증거 요약:** [중영 이중언어 전문가 요약](output/pdf/neorepro_expert_brief_bilingual.pdf)과 [독립 코호트 확장 요약](reports/extension_summary.md). - **고정 결과 재현:** 아래 재현 절의 명령을 사용하십시오. +- **자체 예측기 벤치마크:** `neorepro benchmark predictions.csv --output-dir neorepro-results`를 실행하고 [표준 단일 파일 계약](contracts/README.md)을 참조하십시오. - **제3자 데이터셋 또는 예측기 추가:** [plug-in contract](contracts/README.md)를 참조하십시오. - **고정 버전 인용:** [CITATION.cff](CITATION.cff)와 [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)를 참조하십시오. diff --git a/README.md b/README.md index 6e79599..62ea6c9 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ This is a benchmark/resource contribution, not a new predictor and not a claim o - **Current manuscript:** [resource-positioned manuscript](paper/manuscript_resource.md). - **Two-minute evidence summary:** [bilingual expert brief](output/pdf/neorepro_expert_brief_bilingual.pdf) and [independent-cohort extension summary](reports/extension_summary.md). - **Reproduce the frozen results:** use the command in the Reproduce section below. +- **Benchmark your own predictor:** run `neorepro benchmark predictions.csv --output-dir neorepro-results`; see the [standard one-file contract](contracts/README.md). - **Add a third-party dataset or predictor:** follow the [plug-in contract](contracts/README.md). - **Cite a fixed version:** [CITATION.cff](CITATION.cff) and the [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0). diff --git a/README.zh-CN.md b/README.zh-CN.md index 037b92b..b7cd462 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -15,6 +15,7 @@ NeoRepro 是面向公开 MHC-I 肽–HLA 新抗原预测器的泄漏感知、患 - **当前论文稿：**[资源定位版论文](paper/manuscript_resource.md)。 - **两分钟证据摘要：**[中英双语专家简报](output/pdf/neorepro_expert_brief_bilingual.pdf)和[独立队列扩展摘要](reports/extension_summary.md)。 - **复现冻结结果：**使用下方“复现”部分中的命令。 +- **测试你自己的预测器：**运行 `neorepro benchmark predictions.csv --output-dir neorepro-results`；参见[标准单文件契约](contracts/README.md)。 - **接入第三方数据集或预测器：**请参阅 [plug-in contract](contracts/README.md)。 - **引用固定版本：**参见 [CITATION.cff](CITATION.cff) 和 [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)。 diff --git a/README.zh-TW.md b/README.zh-TW.md index aa20581..000c39b 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -15,6 +15,7 @@ NeoRepro 是一套針對公開 MHC-I 胜肽–HLA 新抗原預測器、考量資 - **目前論文稿：**[資源定位版論文](paper/manuscript_resource.md)。 - **兩分鐘證據摘要：**[中英雙語專家簡報](output/pdf/neorepro_expert_brief_bilingual.pdf)與[獨立隊列延伸摘要](reports/extension_summary.md)。 - **重現凍結結果：**使用下方「重現」章節中的指令。 +- **測試你自己的預測器：**執行 `neorepro benchmark predictions.csv --output-dir neorepro-results`；請參閱[標準單一檔案契約](contracts/README.md)。 - **接入第三方資料集或預測器：**請參閱 [plug-in contract](contracts/README.md)。 - **引用固定版本：**請參閱 [CITATION.cff](CITATION.cff) 與 [v0.1.0 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.0)。 diff --git a/contracts/README.md b/contracts/README.md index 2e67bad..c845311 100644 --- a/contracts/README.md +++ b/contracts/README.md @@ -1,6 +1,37 @@ # NeoRepro plug-in contract -Third parties can validate cards and artifacts, then evaluate in one command: +## Standard one-file entry point + +The fastest public interface accepts one UTF-8 CSV and writes both a machine-readable +`evaluation.json` and a human-readable `report.md`: + +```bash +python -m pip install . +neorepro benchmark predictions.csv --output-dir neorepro-results +``` + +Required columns: + +```text +record_id,patient_id,study_id,label,score,predictor +``` + +Each record's truth metadata must be identical across predictors. `label` is `0` or `1`, and +larger scores are better unless `score_direction` declares `lower`. Blank scores are counted as +unsupported. Optional columns are `score_direction`, `status`, `training_overlap`, `hla`, `assay` +and `cancer_type`. `training_overlap` accepts `exact`, `none` or `unknown`; exact overlaps are +excluded from the primary common-support comparison, while unknown remains unknown. + +The command reports AUROC, AUPRC, fixed-threshold classification metrics, eligible Brier score, +tie-aware patient Recall/Precision/HitRate/NDCG at K=5/10/20, MRR, patient bootstrap confidence +intervals, paired predictor differences, support-matched random ranking and available study/HLA/ +assay/cancer-type strata. Stratified views are descriptive, not held-out validation. + +See `contracts/synthetic/standard_predictions.csv` for a complete two-predictor example. + +## Card and separate-artifact interface + +Third parties can also validate cards and separate artifacts before evaluation: ```bash neorepro dataset validate contracts/dataset-card.example.json diff --git a/contracts/synthetic/README.md b/contracts/synthetic/README.md index 83bb108..9043a4c 100644 --- a/contracts/synthetic/README.md +++ b/contracts/synthetic/README.md @@ -1,3 +1,10 @@ # Synthetic plug-in example This four-row fixture demonstrates explicit unsupported output, common-support evaluation, and patient-level ranking without using biological data. + +Run the portable standard entry point with: + +```bash +neorepro benchmark contracts/synthetic/standard_predictions.csv \ + --output-dir neorepro-results --bootstrap 100 +``` diff --git a/contracts/synthetic/standard_predictions.csv b/contracts/synthetic/standard_predictions.csv new file mode 100644 index 0000000..fb76a73 --- /dev/null +++ b/contracts/synthetic/standard_predictions.csv @@ -0,0 +1,9 @@ +record_id,patient_id,study_id,label,score,predictor,score_direction,status,training_overlap,hla +s1,P1,SYNTHETIC,1,0.9,Example-A,higher,predicted,none,HLA-A*02:01 +s2,P1,SYNTHETIC,0,0.1,Example-A,higher,predicted,none,HLA-A*02:01 +s3,P2,SYNTHETIC,1,0.8,Example-A,higher,predicted,unknown,HLA-A*02:01 +s4,P2,SYNTHETIC,0,0.2,Example-A,higher,predicted,none,HLA-A*02:01 +s1,P1,SYNTHETIC,1,0.7,Example-B,higher,predicted,none,HLA-A*02:01 +s2,P1,SYNTHETIC,0,0.3,Example-B,higher,predicted,none,HLA-A*02:01 +s3,P2,SYNTHETIC,1,0.6,Example-B,higher,predicted,unknown,HLA-A*02:01 +s4,P2,SYNTHETIC,0,,Example-B,higher,unsupported,none,HLA-A*02:01 diff --git a/i18n/readme_state.json b/i18n/readme_state.json index 7060526..0b6d2a6 100644 --- a/i18n/readme_state.json +++ b/i18n/readme_state.json @@ -8,7 +8,7 @@ "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "start-here": "317be5bba5a46c0cb8bdc1917e37aafd70a331f78b7b90b394395365951dd79b", "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, @@ -20,7 +20,7 @@ "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "start-here": "317be5bba5a46c0cb8bdc1917e37aafd70a331f78b7b90b394395365951dd79b", "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, @@ -32,7 +32,7 @@ "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "start-here": "317be5bba5a46c0cb8bdc1917e37aafd70a331f78b7b90b394395365951dd79b", "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, @@ -44,7 +44,7 @@ "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "start-here": "317be5bba5a46c0cb8bdc1917e37aafd70a331f78b7b90b394395365951dd79b", "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, @@ -56,7 +56,7 @@ "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "start-here": "317be5bba5a46c0cb8bdc1917e37aafd70a331f78b7b90b394395365951dd79b", "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, @@ -68,7 +68,7 @@ "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "start-here": "317be5bba5a46c0cb8bdc1917e37aafd70a331f78b7b90b394395365951dd79b", "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, @@ -80,12 +80,12 @@ "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", "preamble": "43db600bf39375ac39feb75e14d3cd34196c564b38589662a408486fedee6bcd", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "c1cb9ae16debaf754ae100ac78b3978baf7da583bcc496bbd55938a972be03cc", + "start-here": "317be5bba5a46c0cb8bdc1917e37aafd70a331f78b7b90b394395365951dd79b", "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } } }, "source": "README.md", - "source_sha256": "e2a1c7940616892f555199132a46c827471183d54bd0c3703a24273897a2ae1f", + "source_sha256": "65a27c828440b921b0fdce482d85a0ac0dd5e975d69703027ffb78ceace14ea2", "version": 1 } diff --git a/src/neorepro/benchmark.py b/src/neorepro/benchmark.py new file mode 100644 index 0000000..89f6762 --- /dev/null +++ b/src/neorepro/benchmark.py @@ -0,0 +1,615 @@ +"""Portable, dependency-free benchmark entry point for external predictions.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +import platform +import random +from collections import defaultdict +from dataclasses import dataclass +from itertools import combinations +from pathlib import Path +from statistics import mean + +from neorepro.metrics import auroc, average_precision, tie_aware_ranking_metrics + +REQUIRED_COLUMNS = ("record_id", "patient_id", "study_id", "label", "score", "predictor") +OPTIONAL_COLUMNS = ( + "score_direction", + "status", + "training_overlap", + "hla", + "assay", + "cancer_type", +) +VALID_DIRECTIONS = {"higher", "lower"} +VALID_STATUSES = {"predicted", "unsupported", "failed", "invalid"} +VALID_OVERLAP = {"exact", "none", "unknown"} +KS = (5, 10, 20) +PATIENT_METRICS = ("mrr",) + tuple( + f"{metric}@{k}" + for k in KS + for metric in ("recall", "precision", "hitrate", "ndcg") +) + + +class BenchmarkError(ValueError): + """Raised when a standard prediction submission is invalid.""" + + +@dataclass(frozen=True) +class Record: + record_id: str + patient_id: str + study_id: str + label: int + hla: str + assay: str + cancer_type: str + + +@dataclass(frozen=True) +class Submission: + path: Path + records: dict[str, Record] + scores: dict[str, dict[str, float | None]] + statuses: dict[str, dict[str, str]] + overlaps: dict[str, dict[str, str]] + directions: dict[str, str] + rows: int + columns: tuple[str, ...] + sha256: str + + +def _percentile(values: list[float], probability: float) -> float: + ordered = sorted(values) + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _interval(values: list[float]) -> dict[str, float] | None: + if not values: + return None + return {"low": _percentile(values, 0.025), "high": _percentile(values, 0.975)} + + +def _parse_label(value: str, line: int) -> int: + if value.strip() not in {"0", "1"}: + raise BenchmarkError(f"line {line}: label must be 0 or 1") + return int(value) + + +def read_submission(path: Path, default_direction: str = "higher") -> Submission: + """Read and strictly validate one portable benchmark CSV.""" + if default_direction not in VALID_DIRECTIONS: + raise BenchmarkError("default score direction must be higher or lower") + try: + raw = path.read_bytes() + except OSError as error: + raise BenchmarkError(f"cannot read {path}: {error}") from error + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError as error: + raise BenchmarkError("prediction CSV must be UTF-8 encoded") from error + + reader = csv.DictReader(text.splitlines()) + if reader.fieldnames is None: + raise BenchmarkError("prediction CSV has no header") + columns = tuple(reader.fieldnames) + missing = [column for column in REQUIRED_COLUMNS if column not in columns] + if missing: + raise BenchmarkError("prediction CSV missing columns: " + ", ".join(missing)) + + records: dict[str, Record] = {} + optional_values: dict[str, dict[str, str]] = defaultdict(dict) + scores: dict[str, dict[str, float | None]] = defaultdict(dict) + statuses: dict[str, dict[str, str]] = defaultdict(dict) + overlaps: dict[str, dict[str, str]] = defaultdict(dict) + directions: dict[str, str] = {} + rows = 0 + for line, row in enumerate(reader, start=2): + rows += 1 + record_id = row["record_id"].strip() + patient_id = row["patient_id"].strip() + study_id = row["study_id"].strip() + predictor = row["predictor"].strip() + if not all((record_id, patient_id, study_id, predictor)): + raise BenchmarkError( + f"line {line}: record_id, patient_id, study_id and predictor are required" + ) + label = _parse_label(row["label"], line) + metadata = { + field: row.get(field, "").strip() for field in ("hla", "assay", "cancer_type") + } + if record_id in records: + existing = records[record_id] + if (existing.patient_id, existing.study_id, existing.label) != ( + patient_id, + study_id, + label, + ): + raise BenchmarkError(f"line {line}: conflicting truth metadata for {record_id}") + for field, value in metadata.items(): + previous = optional_values[record_id].get(field, "") + if previous and value and previous != value: + raise BenchmarkError( + f"line {line}: conflicting {field} metadata for {record_id}" + ) + if value: + optional_values[record_id][field] = value + else: + optional_values[record_id] = metadata + records[record_id] = Record( + record_id=record_id, + patient_id=patient_id, + study_id=study_id, + label=label, + hla=metadata["hla"], + assay=metadata["assay"], + cancer_type=metadata["cancer_type"], + ) + + if record_id in scores[predictor]: + raise BenchmarkError(f"line {line}: duplicate predictor/record_id pair") + direction = row.get("score_direction", "").strip() or default_direction + if direction not in VALID_DIRECTIONS: + raise BenchmarkError(f"line {line}: score_direction must be higher or lower") + if predictor in directions and directions[predictor] != direction: + raise BenchmarkError(f"line {line}: inconsistent score direction for {predictor}") + directions[predictor] = direction + + score_text = row["score"].strip() + status = row.get("status", "").strip() or ("predicted" if score_text else "unsupported") + if status not in VALID_STATUSES: + raise BenchmarkError(f"line {line}: invalid status {status!r}") + score: float | None = None + if status == "predicted": + try: + score = float(score_text) + except ValueError as error: + raise BenchmarkError(f"line {line}: predicted score must be numeric") from error + if not math.isfinite(score): + raise BenchmarkError(f"line {line}: predicted score must be finite") + elif score_text: + raise BenchmarkError(f"line {line}: non-predicted rows must have a blank score") + + overlap = row.get("training_overlap", "").strip() or "unknown" + if overlap not in VALID_OVERLAP: + raise BenchmarkError( + f"line {line}: training_overlap must be exact, none, or unknown" + ) + scores[predictor][record_id] = score + statuses[predictor][record_id] = status + overlaps[predictor][record_id] = overlap + + if not rows: + raise BenchmarkError("prediction CSV has no data rows") + + # Rebuild records after nonblank optional values have been reconciled across predictors. + records = { + record_id: Record( + record_id=record.record_id, + patient_id=record.patient_id, + study_id=record.study_id, + label=record.label, + hla=optional_values[record_id].get("hla", ""), + assay=optional_values[record_id].get("assay", ""), + cancer_type=optional_values[record_id].get("cancer_type", ""), + ) + for record_id, record in records.items() + } + return Submission( + path=path, + records=records, + scores=dict(scores), + statuses=dict(statuses), + overlaps=dict(overlaps), + directions=directions, + rows=rows, + columns=columns, + sha256=hashlib.sha256(raw).hexdigest(), + ) + + +def _oriented(score: float, direction: str) -> float: + return score if direction == "higher" else -score + + +def _classification_metrics( + labels: list[int], raw_scores: list[float], direction: str, threshold: float +) -> dict[str, object]: + oriented = [_oriented(score, direction) for score in raw_scores] + predicted = [ + int(score >= threshold) if direction == "higher" else int(score <= threshold) + for score in raw_scores + ] + tp = sum(label == 1 and call == 1 for label, call in zip(labels, predicted, strict=True)) + tn = sum(label == 0 and call == 0 for label, call in zip(labels, predicted, strict=True)) + fp = sum(label == 0 and call == 1 for label, call in zip(labels, predicted, strict=True)) + fn = sum(label == 1 and call == 0 for label, call in zip(labels, predicted, strict=True)) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + denominator = math.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + result: dict[str, object] = { + "records": len(labels), + "positives": sum(labels), + "prevalence": sum(labels) / len(labels), + "threshold": threshold, + "confusion_matrix": {"tp": tp, "tn": tn, "fp": fp, "fn": fn}, + "precision": precision, + "recall": recall, + "f1": f1, + "mcc": (tp * tn - fp * fn) / denominator if denominator else None, + } + if len(set(labels)) == 2: + result["auroc"] = auroc(labels, oriented) + else: + result["auroc"] = None + result["average_precision"] = average_precision(labels, oriented) if sum(labels) else None + if direction == "higher" and all(0 <= score <= 1 for score in raw_scores): + result["brier_score"] = mean( + (score - label) ** 2 for label, score in zip(labels, raw_scores, strict=True) + ) + result["brier_status"] = "computed_from_probability_range_scores" + else: + result["brier_score"] = None + result["brier_status"] = "not_computed_scores_are_not_higher_is_better_probabilities" + return result + + +def _patient_values( + record_ids: set[str], submission: Submission, predictor: str +) -> dict[str, dict[str, float]]: + by_patient: dict[str, list[str]] = defaultdict(list) + for record_id in sorted(record_ids): + by_patient[submission.records[record_id].patient_id].append(record_id) + values = {} + for patient_id, patient_records in by_patient.items(): + labels = [submission.records[record_id].label for record_id in patient_records] + if not sum(labels): + continue + scores = [submission.scores[predictor][record_id] for record_id in patient_records] + if any(score is None for score in scores): + raise AssertionError("patient metrics require numeric common-support scores") + direction = submission.directions[predictor] + values[patient_id] = tie_aware_ranking_metrics( + labels, + [_oriented(float(score), direction) for score in scores], + KS, + ) + return values + + +def _patient_summary( + values: dict[str, dict[str, float]], bootstrap: int, seed: int +) -> dict[str, object]: + if not values: + return {"eligible_positive_bearing_patients": 0, "metrics": {}} + patient_ids = sorted(values) + samples: dict[str, list[float]] = {metric: [] for metric in PATIENT_METRICS} + if bootstrap: + rng = random.Random(seed) + for _ in range(bootstrap): + draw = rng.choices(patient_ids, k=len(patient_ids)) + for metric in PATIENT_METRICS: + samples[metric].append(mean(values[patient_id][metric] for patient_id in draw)) + return { + "eligible_positive_bearing_patients": len(patient_ids), + "bootstrap_unit": "patient", + "bootstrap_replicates": bootstrap, + "seed": seed, + "metrics": { + metric: { + "estimate": mean(patient[metric] for patient in values.values()), + "ci95": _interval(samples[metric]), + } + for metric in PATIENT_METRICS + }, + } + + +def _paired_differences( + patient_values: dict[str, dict[str, dict[str, float]]], bootstrap: int, seed: int +) -> list[dict[str, object]]: + results = [] + for pair_index, (left, right) in enumerate(combinations(sorted(patient_values), 2)): + patient_ids = sorted(set(patient_values[left]) & set(patient_values[right])) + if not patient_ids: + continue + rng = random.Random(seed + pair_index) + samples: dict[str, list[float]] = {metric: [] for metric in PATIENT_METRICS} + if bootstrap: + for _ in range(bootstrap): + draw = rng.choices(patient_ids, k=len(patient_ids)) + for metric in PATIENT_METRICS: + samples[metric].append( + mean( + patient_values[left][patient_id][metric] + - patient_values[right][patient_id][metric] + for patient_id in draw + ) + ) + results.append( + { + "left": left, + "right": right, + "eligible_paired_patients": len(patient_ids), + "difference": { + metric: { + "estimate": mean( + patient_values[left][patient_id][metric] + - patient_values[right][patient_id][metric] + for patient_id in patient_ids + ), + "ci95": _interval(samples[metric]), + } + for metric in PATIENT_METRICS + }, + } + ) + return results + + +def _stratified_views( + record_ids: set[str], submission: Submission, predictor: str +) -> dict[str, list[dict[str, object]]]: + result = {} + direction = submission.directions[predictor] + for field in ("study_id", "hla", "assay", "cancer_type"): + grouped: dict[str, list[str]] = defaultdict(list) + for record_id in record_ids: + value = getattr(submission.records[record_id], field) + if value: + grouped[value].append(record_id) + views = [] + for value, ids in sorted(grouped.items()): + labels = [submission.records[record_id].label for record_id in ids] + raw_scores = [submission.scores[predictor][record_id] for record_id in ids] + scores = [_oriented(float(score), direction) for score in raw_scores] + views.append( + { + "group": value, + "records": len(ids), + "patients": len( + {submission.records[record_id].patient_id for record_id in ids} + ), + "positives": sum(labels), + "auroc": auroc(labels, scores) if len(set(labels)) == 2 else None, + "average_precision": average_precision(labels, scores) if sum(labels) else None, + } + ) + if views: + result[field] = views + return result + + +def evaluate_submission( + path: Path, + *, + threshold: float = 0.5, + bootstrap: int = 1000, + seed: int = 20260820, + rank_unit: str = "pMHC", + default_direction: str = "higher", +) -> dict[str, object]: + """Evaluate a standard joined prediction file on leakage-filtered common support.""" + if not math.isfinite(threshold): + raise BenchmarkError("threshold must be finite") + if bootstrap < 0: + raise BenchmarkError("bootstrap must be non-negative") + if rank_unit not in {"pMHC", "peptide"}: + raise BenchmarkError("rank unit must be pMHC or peptide") + if rank_unit == "peptide": + raise BenchmarkError( + "peptide ranking requires an explicit cross-HLA aggregation rule; use pMHC" + ) + submission = read_submission(path, default_direction) + predictors = sorted(submission.scores) + universe = set(submission.records) + support = { + predictor: { + record_id + for record_id, score in submission.scores[predictor].items() + if score is not None + } + for predictor in predictors + } + exact = { + predictor: { + record_id + for record_id, overlap in submission.overlaps[predictor].items() + if overlap == "exact" + } + for predictor in predictors + } + audited_support = { + predictor: support[predictor] - exact[predictor] for predictor in predictors + } + raw_common = set.intersection(*(support[predictor] for predictor in predictors)) + common = set.intersection(*(audited_support[predictor] for predictor in predictors)) + if not common: + raise BenchmarkError("no leakage-filtered common support remains for evaluation") + + models = {} + patient_values = {} + for predictor in predictors: + labels = [submission.records[record_id].label for record_id in sorted(common)] + scores = [float(submission.scores[predictor][record_id]) for record_id in sorted(common)] + values = _patient_values(common, submission, predictor) + patient_values[predictor] = values + status_counts = { + status: sum(value == status for value in submission.statuses[predictor].values()) + for status in sorted(VALID_STATUSES) + } + overlap_counts = { + overlap: sum(value == overlap for value in submission.overlaps[predictor].values()) + for overlap in sorted(VALID_OVERLAP) + } + models[predictor] = { + "score_direction": submission.directions[predictor], + "coverage": len(support[predictor]) / len(universe), + "submitted_records": len(submission.scores[predictor]), + "omitted_records": len(universe - set(submission.scores[predictor])), + "supported_records": len(support[predictor]), + "status_counts": status_counts, + "training_overlap": overlap_counts, + "exact_overlap_excluded": len(exact[predictor]), + "pooled_common_support": _classification_metrics( + labels, scores, submission.directions[predictor], threshold + ), + "patient_common_support": _patient_summary(values, bootstrap, seed), + "stratified_descriptive": _stratified_views(common, submission, predictor), + } + + random_values = {} + by_patient: dict[str, list[str]] = defaultdict(list) + for record_id in common: + by_patient[submission.records[record_id].patient_id].append(record_id) + for patient_id, record_ids in by_patient.items(): + labels = [submission.records[record_id].label for record_id in record_ids] + if sum(labels): + random_values[patient_id] = tie_aware_ranking_metrics( + labels, [0.0] * len(labels), KS + ) + + return { + "schema_version": 1, + "protocol": { + "score_direction": "declared_per_predictor; default higher", + "classification_threshold": threshold, + "rank_unit": rank_unit, + "top_k": list(KS), + "tie_rule": "analytic expectation over tied-score permutations", + "primary_comparison": "exact-overlap-filtered common support", + "bootstrap": { + "unit": "patient", + "replicates": bootstrap, + "seed": seed, + }, + "interpretation": ( + "descriptive discrimination and prioritization; not clinical efficacy or " + "held-out generalization" + ), + }, + "input": { + "path": str(path), + "sha256": submission.sha256, + "rows": submission.rows, + "records": len(universe), + "predictors": predictors, + "columns": list(submission.columns), + }, + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + }, + "support": { + "raw_common_records": len(raw_common), + "leakage_filtered_common_records": len(common), + "leakage_filtered_common_coverage": len(common) / len(universe), + }, + "models": models, + "support_matched_random_ranking": _patient_summary(random_values, bootstrap, seed), + "paired_patient_differences": _paired_differences(patient_values, bootstrap, seed), + "limitations": [ + "Unknown training overlap remains unknown and is not evidence of independence.", + "Stratified results are descriptive; this command does not fit held-out folds.", + "Brier score is computed only for higher-is-better scores entirely in [0, 1].", + "Clinical efficacy cannot be inferred from predictor benchmark performance.", + ], + } + + +def markdown_report(result: dict[str, object]) -> str: + """Render a concise, self-contained human-readable benchmark report.""" + support = result["support"] + protocol = result["protocol"] + lines = [ + "# NeoRepro standard benchmark report", + "", + f"Input SHA-256: `{result['input']['sha256']}`", + "", + ( + f"Primary comparison uses **{support['leakage_filtered_common_records']}** " + f"records ({support['leakage_filtered_common_coverage']:.1%}) on " + "exact-overlap-filtered common support." + ), + "", + "| Predictor | Coverage | AUROC | AUPRC | Recall@20 | NDCG@5 |", + "|---|---:|---:|---:|---:|---:|", + ] + for predictor, model in result["models"].items(): + pooled = model["pooled_common_support"] + patient = model["patient_common_support"]["metrics"] + + def value(metric: object) -> str: + return "NA" if metric is None else f"{float(metric):.3f}" + + lines.append( + f"| {predictor} | {model['coverage']:.1%} | {value(pooled['auroc'])} | " + f"{value(pooled['average_precision'])} | " + f"{value(patient.get('recall@20', {}).get('estimate'))} | " + f"{value(patient.get('ndcg@5', {}).get('estimate'))} |" + ) + lines += [ + "", + "## Evaluation contract", + "", + f"- Classification threshold: `{protocol['classification_threshold']}`.", + f"- Ranking unit: `{protocol['rank_unit']}`; Top-K: `{protocol['top_k']}`.", + f"- Tie handling: {protocol['tie_rule']}.", + ( + f"- Patient bootstrap: {protocol['bootstrap']['replicates']} replicates, " + f"seed `{protocol['bootstrap']['seed']}`." + ), + "- Missing or failed predictions are reported and never imputed.", + "- Exact declared training overlaps are excluded; unknown overlap remains unknown.", + "", + "## Interpretation limits", + "", + ] + lines.extend(f"- {limitation}" for limitation in result["limitations"]) + return "\n".join(lines) + "\n" + + +def run_benchmark( + path: Path, + output_dir: Path, + *, + threshold: float = 0.5, + bootstrap: int = 1000, + seed: int = 20260820, + rank_unit: str = "pMHC", + default_direction: str = "higher", +) -> tuple[dict[str, object], Path, Path]: + """Evaluate one submission and atomically publish JSON and Markdown outputs.""" + result = evaluate_submission( + path, + threshold=threshold, + bootstrap=bootstrap, + seed=seed, + rank_unit=rank_unit, + default_direction=default_direction, + ) + output_dir.mkdir(parents=True, exist_ok=True) + json_path = output_dir / "evaluation.json" + report_path = output_dir / "report.md" + json_temp = output_dir / ".evaluation.json.tmp" + report_temp = output_dir / ".report.md.tmp" + json_temp.write_text( + json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + report_temp.write_text(markdown_report(result), encoding="utf-8") + json_temp.replace(json_path) + report_temp.replace(report_path) + return result, json_path, report_path diff --git a/src/neorepro/cli.py b/src/neorepro/cli.py index f776cb7..6db4f9c 100644 --- a/src/neorepro/cli.py +++ b/src/neorepro/cli.py @@ -8,6 +8,7 @@ from pathlib import Path from neorepro.audit import AuditError, audit_predictions +from neorepro.benchmark import BenchmarkError, run_benchmark from neorepro.contract import ( ContractError, evaluate, @@ -65,6 +66,25 @@ def main(argv: list[str] | None = None) -> int: ), ) audit_parser.add_argument("predictions", type=Path, help="five-column prediction CSV") + benchmark_parser = subparsers.add_parser( + "benchmark", + help="run the portable standard benchmark from one joined prediction CSV", + description=( + "Evaluate one or more predictors from a CSV containing record_id, patient_id, " + "study_id, label, score and predictor. Produces evaluation.json and report.md." + ), + ) + benchmark_parser.add_argument("predictions", type=Path) + benchmark_parser.add_argument( + "--output-dir", type=Path, default=Path("neorepro-results") + ) + benchmark_parser.add_argument("--threshold", type=float, default=0.5) + benchmark_parser.add_argument("--bootstrap", type=int, default=1000) + benchmark_parser.add_argument("--seed", type=int, default=20260820) + benchmark_parser.add_argument( + "--score-direction", choices=["higher", "lower"], default="higher" + ) + benchmark_parser.add_argument("--rank-unit", choices=["pMHC", "peptide"], default="pMHC") def add_path_command(name, help_text): p = subparsers.add_parser(name, help=help_text) @@ -111,6 +131,35 @@ def add_path_command(name, help_text): parser.error(str(error)) print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) return 0 + if args.command == "benchmark": + try: + result, json_path, report_path = run_benchmark( + args.predictions, + args.output_dir, + threshold=args.threshold, + bootstrap=args.bootstrap, + seed=args.seed, + rank_unit=args.rank_unit, + default_direction=args.score_direction, + ) + except (BenchmarkError, OSError) as error: + parser.error(str(error)) + print( + json.dumps( + { + "status": "ok", + "predictors": result["input"]["predictors"], + "common_support": result["support"][ + "leakage_filtered_common_records" + ], + "evaluation": str(json_path), + "report": str(report_path), + }, + indent=2, + sort_keys=True, + ) + ) + return 0 try: if args.command == "dataset": print(json.dumps(validate_card(args.path, "dataset"), indent=2)) diff --git a/tests/test_standard_benchmark.py b/tests/test_standard_benchmark.py new file mode 100644 index 0000000..158f79b --- /dev/null +++ b/tests/test_standard_benchmark.py @@ -0,0 +1,74 @@ +import json +from pathlib import Path + +import pytest + +from neorepro.benchmark import BenchmarkError, evaluate_submission, run_benchmark +from neorepro.cli import main + +ROOT = Path(__file__).resolve().parents[1] +EXAMPLE = ROOT / "contracts/synthetic/standard_predictions.csv" + + +def test_standard_benchmark_reports_full_contract(tmp_path: Path) -> None: + result, json_path, report_path = run_benchmark( + EXAMPLE, tmp_path / "result", bootstrap=50, seed=7 + ) + + assert result["support"]["raw_common_records"] == 3 + assert result["support"]["leakage_filtered_common_records"] == 3 + assert result["models"]["Example-A"]["pooled_common_support"]["auroc"] == 1.0 + assert result["models"]["Example-A"]["patient_common_support"]["metrics"][ + "recall@20" + ]["estimate"] == 1.0 + assert result["models"]["Example-B"]["status_counts"]["unsupported"] == 1 + assert result["support_matched_random_ranking"]["metrics"]["ndcg@5"] + assert len(result["paired_patient_differences"]) == 1 + assert json.loads(json_path.read_text())["schema_version"] == 1 + assert "NeoRepro standard benchmark report" in report_path.read_text() + + +def test_exact_overlap_is_removed_from_common_support(tmp_path: Path) -> None: + text = EXAMPLE.read_text().replace( + "s1,P1,SYNTHETIC,1,0.9,Example-A,higher,predicted,none", + "s1,P1,SYNTHETIC,1,0.9,Example-A,higher,predicted,exact", + ) + path = tmp_path / "overlap.csv" + path.write_text(text) + + result = evaluate_submission(path, bootstrap=0) + + assert result["models"]["Example-A"]["exact_overlap_excluded"] == 1 + assert result["support"]["leakage_filtered_common_records"] == 2 + + +def test_standard_benchmark_rejects_conflicting_truth(tmp_path: Path) -> None: + path = tmp_path / "bad.csv" + path.write_text( + "record_id,patient_id,study_id,label,score,predictor\n" + "r1,p1,s1,1,0.9,A\n" + "r1,p1,s1,0,0.8,B\n" + ) + + with pytest.raises(BenchmarkError, match="conflicting truth"): + evaluate_submission(path) + + +def test_standard_benchmark_cli_writes_outputs( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + output_dir = tmp_path / "cli" + assert main( + [ + "benchmark", + str(EXAMPLE), + "--output-dir", + str(output_dir), + "--bootstrap", + "10", + ] + ) == 0 + summary = json.loads(capsys.readouterr().out) + assert summary["status"] == "ok" + assert (output_dir / "evaluation.json").exists() + assert (output_dir / "report.md").exists()