diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..51d32f2 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu + +export PYTHONUTF8=1 +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 + +expected_name="stevezkw" +author_ident="$(git var GIT_AUTHOR_IDENT)" +committer_ident="$(git var GIT_COMMITTER_IDENT)" + +case "$author_ident" in + "$expected_name <"*) ;; + *) + echo "pre-commit: author must be '$expected_name'; got: $author_ident" >&2 + exit 1 + ;; +esac + +case "$committer_ident" in + "$expected_name <"*) ;; + *) + echo "pre-commit: committer must be '$expected_name'; got: $committer_ident" >&2 + exit 1 + ;; +esac + +git diff --cached --check + +changed_paths="$(git diff --cached --name-only --diff-filter=ACMR)" + +if printf '%s\n' "$changed_paths" | grep -Eq '^\.githooks/(pre-commit|pre-push)$'; then + sh -n .githooks/pre-commit .githooks/pre-push +fi + +if printf '%s\n' "$changed_paths" | grep -Eq '^(README(\.[^.]+)?\.md|i18n/(glossary\.md|readme_state\.json)|scripts/readme_i18n\.py|tests/test_readme_i18n\.py)$'; then + python3 scripts/readme_i18n.py check + uv run pytest -q tests/test_readme_i18n.py +fi + +if printf '%s\n' "$changed_paths" | grep -Eq '^(src|scripts|tests)/.*\.py$'; then + uv run ruff check src scripts tests +fi + +if printf '%s\n' "$changed_paths" | grep -Eq '^(src/neorepro/(audit|cli|metrics)\.py|tests/test_(audit_cli|metrics)\.py)$'; then + uv run pytest -q tests/test_audit_cli.py tests/test_metrics.py +fi diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..0fe0e58 --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +export PYTHONUTF8=1 +export LC_ALL=en_US.UTF-8 +export LANG=en_US.UTF-8 + +python3 scripts/readme_i18n.py check +uv run pytest -q diff --git a/.gitignore b/.gitignore index 0818c8d..c4b09a8 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ __pycache__/ *.py[cod] .pytest_cache/ .ruff_cache/ +.bootstrap_checkpoint.json +.bootstrap_checkpoint.json.tmp .coverage htmlcov/ build/ @@ -35,6 +37,8 @@ results/raw_predictions/** !results/raw_predictions/improve/baselines/loso/*.csv !results/raw_predictions/zhao/ !results/raw_predictions/zhao/*.csv +!results/raw_predictions/rcc/ +!results/raw_predictions/rcc/*.csv !results/raw_predictions/improve/expanded_9_10/ !results/raw_predictions/improve/expanded_9_10/*.csv results/raw_predictions/improve/peptide_sensitivity/ diff --git a/FINAL_REPORT.md b/FINAL_REPORT.md index 66fc1a3..8d07ee6 100644 --- a/FINAL_REPORT.md +++ b/FINAL_REPORT.md @@ -24,6 +24,12 @@ This cohort is complementary, not a pure biological replication. Peptide-pulsed On the original IMPROVE 9–10mer subset, the expanded model set retained the original direction: PRIME AUROC 0.605, BigMHC 0.547, DeepImmuno-CNN 0.527 on its 11,036 supported records, and DeepHLApan 0.508. Thus the external reversal is not explained merely by adding the two older models; it is specifically a dataset/endpoint-domain change. +## RCC cohort and reusable-resource extension + +A second endpoint-distinct vaccine extension adds 129 individually assayed short peptides from nine RCC vaccine recipients. Its protocol was frozen before prediction, its source workbook is checksum-pinned, and no known exact PRIME2, BigMHC-construction or DeepImmuno training overlap was identified. Because the cohort is small, uses source-predicted HLA assignments and measures post-vaccination in-vitro stimulation, its results are descriptive and do not establish natural presentation, untreated immunogenicity, clinical efficacy or a universal predictor order. All quantitative RCC results and uncertainty intervals are generated into `paper/manuscript_resource.md` from `results/analysis/rcc/metrics.json`. + +The repository now also includes an expanded public-artifact census and a machine-validated extension contract for datasets, predictors and standardized prediction artifacts. Heterogeneous or failed tools remain visible in the registry but are not promoted into the peptide–HLA benchmark. Three-domain stability outputs are explicitly exploratory, task-stratified and conditional on observed support. + ## Robustness and interpretation - Excluding 35 peptides seen in PRIME2 training only under another HLA left the main direction unchanged (BigMHC AUROC 0.545; PRIME 0.596). Excluding 18 additional same-HLA, same-length Hamming-distance-one records did likewise (0.546; 0.596). diff --git a/Makefile b/Makefile index 68cb8c1..0beabcc 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ export LC_ALL := en_US.UTF-8 export LANG := en_US.UTF-8 PY := uv run python -.PHONY: install setup-predictors download-improve build-improve audit-improve extension download-zhao build-zhao audit-zhao predict-zhao evaluate-zhao predict-expanded-improve evaluate-expanded-improve \ +.PHONY: install setup-predictors contract-gate release-boundary predictor-census external-funnel rcc evaluate-rcc stability download-improve build-improve audit-improve extension download-zhao build-zhao audit-zhao predict-zhao evaluate-zhao predict-expanded-improve evaluate-expanded-improve \ predict-improve baselines-improve evaluate-improve hla-improve figures manuscript \ sensitivity-improve build-peptide-sensitivity evaluate-peptide-sensitivity \ evaluate-peptide-hla-rank-sensitivity build-fixed-sensitivities \ @@ -14,6 +14,47 @@ PY := uv run python install: uv sync --extra dev --extra analysis +contract-gate: + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli dataset validate "$(CURDIR)/contracts/dataset-card.example.json" + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli predictor validate "$(CURDIR)/contracts/predictor-card.example.json" + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli artifact "$(CURDIR)/contracts/synthetic/predictions.csv" --benchmark "$(CURDIR)/contracts/synthetic/benchmark.csv" + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest python -m neorepro.cli evaluate "$(CURDIR)/contracts/synthetic/benchmark.csv" "$(CURDIR)/contracts/synthetic/predictions.csv" \ + --output /tmp/neorepro-contract-evaluation.json --report /tmp/neorepro-contract-evaluation.md + cd /tmp && PYTHONPATH="$(CURDIR)/src" uv run --no-project --with pytest pytest "$(CURDIR)/tests/test_contract.py" -q + +release-boundary: + $(PY) scripts/audit_release_boundary.py + +predictor-census: + $(PY) scripts/reproduce_public_predictors.py + +external-funnel: + $(PY) scripts/validate_external_cohort_funnel.py + +rcc: + $(PY) scripts/build_rcc_benchmark.py --input data/raw/rcc_2025_table2.xlsx \ + --output data/processed/rcc_vaccine_benchmark.csv --summary data/rcc_vaccine_summary.json + $(PY) scripts/audit_external_training_overlap.py \ + --benchmark data/processed/rcc_vaccine_benchmark.csv \ + --prime2-archive data/raw/prime2_table_s4.xlsx \ + --output research/training_overlap_audit_rcc.csv \ + --summary research/training_overlap_summary_rcc.json + $(MAKE) evaluate-rcc + +evaluate-rcc: + PYTHONPATH=src $(PY) scripts/evaluate_benchmark.py \ + --benchmark data/processed/rcc_vaccine_benchmark.csv \ + --predictions results/raw_predictions/rcc/*.csv \ + --output-dir results/analysis/rcc --bootstrap 2000 --seed 20260820 + +stability: + $(PY) scripts/analyze_stability.py \ + --benchmark data/processed/improve_benchmark.csv data/processed/zhao_vaccine_benchmark.csv data/processed/rcc_vaccine_benchmark.csv \ + --prediction-dir results/raw_predictions/improve results/raw_predictions/zhao results/raw_predictions/rcc \ + --output-dir results/analysis/stability --bootstrap 2000 --seed 20260820 + $(PY) scripts/render_stability_figure.py + + setup-predictors: $(PY) scripts/setup_predictors.py --accept-academic-licenses @@ -218,7 +259,7 @@ figures: evaluate-improve baselines-improve hla-improve --loso results/analysis/improve/baselines/loso/metrics.json \ --hla results/analysis/improve/hla_sensitivity.csv -manuscript: figures sensitivity-improve +manuscript: figures sensitivity-improve evaluate-rcc stability external-funnel contract-gate $(PY) scripts/build_manuscript.py manifest: manuscript validate-metrics @@ -232,7 +273,7 @@ validate-metrics: evaluate-improve baselines-improve sensitivity-improve reproduce-results: install $(MAKE) verify-reproduction -verify-reproduction: manifest +verify-reproduction: manifest release-boundary $(MAKE) test # Also download source data and install/run licensed third-party predictors. diff --git a/README.de.md b/README.de.md index 6157b25..0e7bc7e 100644 --- a/README.de.md +++ b/README.de.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[Wir suchen aktiv externe Gutachterinnen und Gutachter sowie Autorinnen und Autoren von Prädiktoren.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> Plausibilitätsprüfungen von 15–30 Minuten, Reproduktionsversuche, Datensatzvorschläge und kritische Gegenprüfungen sind willkommen. + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro ist eine reproduzierbare, patientenbezogene Benchmark-Ressource für öffentliche MHC-I-Peptid–HLA-Neoantigen-Prädiktoren unter Berücksichtigung von Datenlecks. Sie bündelt festgeschriebene Prädiktor-Artefakte, Provenienz auf Datensatzebene, Prüfungen auf Überschneidungen mit Trainingsdaten, Vergleiche auf einer gemeinsamen Auswertungsmenge, patientenbezogene Unsicherheit, an die Abdeckung angepasste Zufallsreferenzen und maschinell erzeugte Ergebnisse. @@ -13,6 +17,8 @@ Der Beitrag ist eine Benchmark- und Forschungsressource, kein neuer Prädiktor u - **Aktuelles Manuskript:** [als Ressource positioniertes Manuskript](paper/manuscript_resource.md). - **Evidenz in zwei Minuten:** [zweisprachige Expertenübersicht](output/pdf/neorepro_expert_brief_bilingual.pdf) und [Zusammenfassung der unabhängigen Kohortenerweiterung](reports/extension_summary.md). - **Festgeschriebene Ergebnisse reproduzieren:** den Befehl im Abschnitt „Reproduktion“ verwenden. +- **Eigenen Prädiktor benchmarken:** `neorepro benchmark predictions.csv --output-dir neorepro-results` ausführen; siehe [Standardvertrag für eine einzelne Datei](contracts/README.md). +- **Drittanbieter-Datensatz oder -Prädiktor hinzufügen:** den [plug-in contract](contracts/README.md) verwenden. - **Feste Version zitieren:** [CITATION.cff](CITATION.cff), [v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1) und [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064). Der wissenschaftliche Vertrag und Umfang stehen in [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -20,10 +26,12 @@ Der wissenschaftliche Vertrag und Umfang stehen in [RESEARCH_SPEC.md](RESEARCH_S ## Status - Aktuelle Literaturprüfung: abgeschlossen; Entscheidung `RESCOPE, then GO` -- Reproduzierte Prädiktoren: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN und DeepHLApan +- Benchmark-Prädiktoren: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN und DeepHLApan; für sieben weitere öffentliche Werkzeuge liegen versionierte Profil-, Nichtvergleichbarkeits- oder Fehlschlagsnachweise vor - TESLA-Pilot: abgeschlossen; als Positivkontrolle für Trainingsdatenüberschneidung neu eingestuft - Primärer Benchmark: IMPROVE, 17,475 nach Leakage-Filterung verbleibende Datensätze, 70 Patienten, 3 Kohorten - Primäre IMPROVE-Inferenz: abgeschlossen; 52,425 Vorhersagen fester Werkzeuge ohne fehlende Datensätze +- Externe Domänen: Zhao-Impfkohorte sowie eine separat festgeschriebene RCC-Impfkohorte mit 129 Datensätzen und 9 Patienten +- Wiederverwendbare Erweiterungsschnittstelle: maschinell validierte Dataset Cards, Predictor Cards und Verträge für Vorhersageartefakte - Manuskript: [Ressourcenfassung](paper/manuscript_resource.md), aus festgeschriebenen Ergebnisdateien erzeugt; unabhängige statistische und biologische Prüfung abgeschlossen ## Hauptergebnis @@ -46,7 +54,7 @@ Unabhängige bootstrap-Analysen werden von Make parallelisiert. Bei begrenzter C - **Prüfbare Ausgaben:** [endgültige Ergebnistabelle](results/final_results.csv), [Abbildungen](results/figures/), [Prüfung auf Trainingsüberschneidungen](research/training_overlap_summary_improve.json) und [SHA-256-Manifest](results/manifest.json). - **Publikationsplanung:** [Strategie für Zielzeitschriften](reports/target_venues_2026-08-20.md). -Die unabhängige Erweiterung um die Zhao 2026-Impfkohorte lässt sich mit `make -j4 extension` reproduzieren. Die kompakte Evidenzübersicht steht in [reports/extension_summary.md](reports/extension_summary.md), der vor der Inferenz festgeschriebene Studienvertrag in [research/extension_protocol.json](research/extension_protocol.json). Der externe Endpunkt ist ELISPOT nach der Impfung und Verabreichung peptidgepulster dendritischer Zellen; er darf nicht als natürliche Tumorpräsentation oder klinische Wirksamkeit interpretiert werden. +Die unabhängige Erweiterung um die Zhao 2026-Impfkohorte lässt sich mit `make -j4 extension` reproduzieren. Die kompakte Evidenzübersicht steht in [reports/extension_summary.md](reports/extension_summary.md), der vor der Inferenz festgeschriebene Studienvertrag in [research/extension_protocol.json](research/extension_protocol.json). Die separat festgeschriebene RCC-Erweiterung steht in [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), explorative Stabilitätsausgaben für drei Domänen unter `results/analysis/stability/`. Beide externen Endpunkte wurden nach der Impfung erhoben und dürfen nicht als natürliche Tumorpräsentation oder klinische Wirksamkeit interpretiert werden. ## Lizenz diff --git a/README.es.md b/README.es.md index f16d09a..858e6f6 100644 --- a/README.es.md +++ b/README.es.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[Buscamos activamente revisores externos y autores de predictores.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> Agradecemos comprobaciones de coherencia de 15–30 minutos, intentos de reproducción, sugerencias de conjuntos de datos y críticas adversariales. + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro es un recurso de evaluación reproducible, sensible a la fuga de datos y a nivel de paciente para predictores públicos de neoantígenos péptido–HLA de MHC-I. Reúne artefactos de predictores fijados, procedencia por registro, auditorías de solapamiento con datos de entrenamiento, comparaciones sobre soporte común, incertidumbre a nivel de paciente, referencias aleatorias ajustadas al soporte y resultados generados automáticamente. @@ -13,6 +17,8 @@ Es una contribución de referencia y recursos, no un predictor nuevo ni una afir - **Manuscrito actual:** [manuscrito orientado como recurso](paper/manuscript_resource.md). - **Resumen de la evidencia en dos minutos:** [informe bilingüe para expertos](output/pdf/neorepro_expert_brief_bilingual.pdf) y [resumen de la extensión con cohorte independiente](reports/extension_summary.md). - **Reproducir los resultados fijados:** usar el comando de la sección «Reproducción». +- **Evaluar su propio predictor:** ejecutar `neorepro benchmark predictions.csv --output-dir neorepro-results`; consultar el [contrato estándar de un solo archivo](contracts/README.md). +- **Añadir un conjunto de datos o predictor de terceros:** consultar el [plug-in contract](contracts/README.md). - **Citar una versión fija:** [CITATION.cff](CITATION.cff), [v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1) y [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064). El contrato científico y el alcance están en [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -20,10 +26,12 @@ El contrato científico y el alcance están en [RESEARCH_SPEC.md](RESEARCH_SPEC. ## Estado - Auditoría de la literatura actual: completa; decisión `RESCOPE, then GO` -- Predictores reproducidos: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN y DeepHLApan +- Predictores evaluados: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN y DeepHLApan; otros siete instrumentos públicos conservan registros versionados de perfil, no comparabilidad o fallo de reproducción - Piloto TESLA: completo; reclasificado como control positivo de solapamiento con entrenamiento - Evaluación principal: IMPROVE, 17,475 registros tras filtrar fugas, 70 pacientes y 3 cohortes - Inferencia principal de IMPROVE: completa; 52,425 predicciones de herramientas fijas sin registros ausentes +- Dominios externos: cohorte de vacunación Zhao y una cohorte RCC fijada por separado con 129 registros y 9 pacientes +- Interfaz de extensión reutilizable: Dataset Cards, Predictor Cards y contratos de artefactos de predicción validados por máquina - Manuscrito: [versión orientada como recurso](paper/manuscript_resource.md), generada a partir de resultados fijados; revisión estadística y biológica independiente completada ## Resultado principal @@ -46,7 +54,7 @@ Make paraleliza los análisis bootstrap independientes. Use `make reproduce-resu - **Resultados auditables:** [tabla final de resultados](results/final_results.csv), [figuras](results/figures/), [auditoría de solapamiento con entrenamiento](research/training_overlap_summary_improve.json) y [manifiesto SHA-256](results/manifest.json). - **Planificación del envío:** [estrategia de revistas objetivo](reports/target_venues_2026-08-20.md). -La extensión independiente con la cohorte de vacunación Zhao 2026 se reproduce con `make -j4 extension`. El resumen conciso de la evidencia está en [reports/extension_summary.md](reports/extension_summary.md), y el contrato congelado antes de la inferencia en [research/extension_protocol.json](research/extension_protocol.json). El criterio externo es ELISPOT posterior a la vacunación tras administrar células dendríticas pulsadas con péptidos; no debe interpretarse como presentación tumoral natural ni eficacia clínica. +La extensión independiente con la cohorte de vacunación Zhao 2026 se reproduce con `make -j4 extension`. El resumen conciso de la evidencia está en [reports/extension_summary.md](reports/extension_summary.md), y el contrato congelado antes de la inferencia en [research/extension_protocol.json](research/extension_protocol.json). La extensión RCC fijada por separado está en [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), y las salidas exploratorias de estabilidad de tres dominios en `results/analysis/stability/`. Ambos criterios externos son posteriores a la vacunación y no deben interpretarse como presentación tumoral natural ni eficacia clínica. ## Licencia diff --git a/README.fr.md b/README.fr.md index 44db46a..2f0f393 100644 --- a/README.fr.md +++ b/README.fr.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[Nous recherchons activement des évaluateurs externes et des auteurs de prédicteurs.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> Les vérifications de cohérence de 15–30 minutes, les tentatives de reproduction, les suggestions de jeux de données et les critiques contradictoires sont les bienvenues. + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro est une ressource d’évaluation reproductible, tenant compte des fuites de données et menée au niveau du patient, pour les prédicteurs publics de néoantigènes peptide–HLA de MHC-I. Elle rassemble des artefacts de prédicteurs épinglés, la provenance de chaque enregistrement, des audits de chevauchement avec les données d’entraînement, des comparaisons sur un support commun, l’incertitude au niveau du patient, des références aléatoires adaptées au support et des résultats générés automatiquement. @@ -13,6 +17,8 @@ Il s’agit d’une contribution de type ressource et benchmark, et non d’un n - **Manuscrit actuel :** [manuscrit positionné comme ressource](paper/manuscript_resource.md). - **Synthèse des preuves en deux minutes :** [note bilingue pour experts](output/pdf/neorepro_expert_brief_bilingual.pdf) et [résumé de l’extension sur une cohorte indépendante](reports/extension_summary.md). - **Reproduire les résultats figés :** utiliser la commande de la section « Reproduction » ci-dessous. +- **Évaluer votre propre prédicteur :** exécuter `neorepro benchmark predictions.csv --output-dir neorepro-results` ; consulter le [contrat standard à fichier unique](contracts/README.md). +- **Ajouter un jeu de données ou un prédicteur tiers :** consulter le [plug-in contract](contracts/README.md). - **Citer une version figée :** [CITATION.cff](CITATION.cff), [v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1) et [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064). Le contrat scientifique et le périmètre sont décrits dans [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -20,10 +26,12 @@ Le contrat scientifique et le périmètre sont décrits dans [RESEARCH_SPEC.md]( ## État - Audit de la littérature actuelle : terminé ; décision `RESCOPE, then GO` -- Prédicteurs reproduits : MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN et DeepHLApan +- Prédicteurs évalués : MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN et DeepHLApan ; sept autres outils publics disposent de dossiers versionnés de profil, de non-comparabilité ou d’échec de reproduction - Pilote TESLA : terminé ; reclassé comme contrôle positif de chevauchement avec l’entraînement - Benchmark principal : IMPROVE, 17,475 enregistrements après filtrage des fuites, 70 patients et 3 cohortes - Inférence IMPROVE principale : terminée ; 52,425 prédictions d’outils fixes sans enregistrement manquant +- Domaines externes : cohorte vaccinale Zhao et cohorte vaccinale RCC figée séparément avec 129 enregistrements et 9 patients +- Interface d’extension réutilisable : Dataset Cards, Predictor Cards et contrats d’artefacts de prédiction validés automatiquement - Manuscrit : [version positionnée comme ressource](paper/manuscript_resource.md), générée à partir des résultats figés ; revues statistique et biologique indépendantes terminées ## Résultat principal @@ -46,7 +54,7 @@ Make parallélise les analyses bootstrap indépendantes. Utilisez `make reproduc - **Sorties auditables :** [table finale des résultats](results/final_results.csv), [figures](results/figures/), [audit du chevauchement avec l’entraînement](research/training_overlap_summary_improve.json) et [manifeste SHA-256](results/manifest.json). - **Planification de la soumission :** [stratégie des revues cibles](reports/target_venues_2026-08-20.md). -L’extension indépendante sur la cohorte vaccinale Zhao 2026 est reproduite avec `make -j4 extension`. La synthèse concise des preuves se trouve dans [reports/extension_summary.md](reports/extension_summary.md), et le contrat figé avant l’inférence dans [research/extension_protocol.json](research/extension_protocol.json). Le critère externe est un ELISPOT après vaccination et administration de cellules dendritiques chargées en peptides ; il ne doit pas être interprété comme une présentation tumorale naturelle ni comme une efficacité clinique. +L’extension indépendante sur la cohorte vaccinale Zhao 2026 est reproduite avec `make -j4 extension`. La synthèse concise des preuves se trouve dans [reports/extension_summary.md](reports/extension_summary.md), et le contrat figé avant l’inférence dans [research/extension_protocol.json](research/extension_protocol.json). L’extension RCC figée séparément se trouve dans [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), et les sorties exploratoires de stabilité sur trois domaines dans `results/analysis/stability/`. Les deux critères externes sont postérieurs à la vaccination et ne doivent pas être interprétés comme une présentation tumorale naturelle ni comme une efficacité clinique. ## Licence diff --git a/README.ja.md b/README.ja.md index e9d5496..55cbccb 100644 --- a/README.ja.md +++ b/README.ja.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[外部レビュアーおよび予測器の作者を積極的に募集しています。](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 15–30 分の妥当性確認、再現の試行、データセットの提案、批判的な検証を歓迎します。 + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro は、公開されている MHC-I ペプチド–HLA ネオアンチゲン予測器を対象とした、リーケージを考慮した患者単位の再現可能なベンチマークリソースです。固定された予測器アーティファクト、レコード単位の来歴、学習データ重複監査、共通評価可能集合での比較、患者単位の不確実性、評価可能範囲を一致させたランダムベースライン、および機械生成結果をまとめています。 @@ -13,6 +17,8 @@ NeoRepro は、公開されている MHC-I ペプチド–HLA ネオアンチゲ - **現行原稿：**[リソースとして位置付けた原稿](paper/manuscript_resource.md)。 - **短時間で読めるエビデンス概要：**[中英併記の専門家向け概要](output/pdf/neorepro_expert_brief_bilingual.pdf)および[独立コホート拡張の概要](reports/extension_summary.md)。 - **凍結済み結果の再現：**下記「再現」節のコマンドを使用してください。 +- **独自予測器のベンチマーク：**`neorepro benchmark predictions.csv --output-dir neorepro-results` を実行し、[標準単一ファイル契約](contracts/README.md)を参照してください。 +- **第三者データセットまたは予測器の追加：**[plug-in contract](contracts/README.md) を参照してください。 - **固定版の引用：**[CITATION.cff](CITATION.cff)、[v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1)、および [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064)を参照してください。 科学的な研究契約と範囲については [RESEARCH_SPEC.md](RESEARCH_SPEC.md) を参照してください。 @@ -20,10 +26,12 @@ NeoRepro は、公開されている MHC-I ペプチド–HLA ネオアンチゲ ## 状況 - 最新文献監査：完了、判断は `RESCOPE, then GO` -- 再現済み予測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN、DeepHLApan +- ベンチマーク対象予測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN、DeepHLApan。さらに七つの公開ツールについて、プロファイル限定、比較不能、または再現失敗の記録をバージョン管理 - TESLA パイロット：完了、学習データ重複の陽性対照として再分類 - 主要ベンチマーク：IMPROVE、リーケージ除外後 17,475 レコード、70 患者、3 コホート - 主要 IMPROVE 推論：完了、固定ツールによる 52,425 予測、欠損レコードなし +- 外部ドメイン：Zhao ワクチンコホートと、別途凍結した 129 レコード・9 患者の RCC ワクチンコホート +- 再利用可能な拡張インターフェース：機械検証済み Dataset Card、Predictor Card、予測アーティファクト契約 - 原稿：[リソース位置付け版](paper/manuscript_resource.md)、凍結結果ファイルから生成、独立した統計学・生物学レビュー済み ## 主な結果 @@ -46,7 +54,7 @@ make -j4 reproduce-results - **監査可能な出力：**[最終結果表](results/final_results.csv)、[図](results/figures/)、[学習データ重複監査](research/training_overlap_summary_improve.json)、[SHA-256 マニフェスト](results/manifest.json)。 - **投稿計画：**[投稿先戦略](reports/target_venues_2026-08-20.md)。 -独立した Zhao 2026 ワクチンコホート拡張は `make -j4 extension` で再現できます。簡潔なエビデンス概要は [reports/extension_summary.md](reports/extension_summary.md)、推論前に凍結した研究契約は [research/extension_protocol.json](research/extension_protocol.json) にあります。外部エンドポイントは、ペプチドをパルスした樹状細胞投与後のワクチン接種後 ELISPOT であり、自然な腫瘍提示や臨床効果を示すものではありません。 +独立した Zhao 2026 ワクチンコホート拡張は `make -j4 extension` で再現できます。簡潔なエビデンス概要は [reports/extension_summary.md](reports/extension_summary.md)、推論前に凍結した研究契約は [research/extension_protocol.json](research/extension_protocol.json) にあります。別途凍結した RCC 拡張は [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json)、三ドメインの探索的安定性出力は `results/analysis/stability/` にあります。両外部エンドポイントはワクチン接種後の測定であり、自然な腫瘍提示や臨床効果を示すものではありません。 ## ライセンス diff --git a/README.ko.md b/README.ko.md index f481dbe..774ace3 100644 --- a/README.ko.md +++ b/README.ko.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[외부 검토자와 예측기 개발자를 적극적으로 찾고 있습니다.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 15–30분의 타당성 점검, 재현 시도, 데이터셋 제안 및 비판적 검토를 환영합니다. + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro는 공개 MHC-I 펩타이드–HLA 신생항원 예측기를 위한 데이터 누출 인지형, 환자 수준, 재현 가능한 벤치마크 리소스입니다. 고정된 예측기 아티팩트, 레코드 수준 출처, 학습 데이터 중복 감사, 공통 평가 가능 집합 비교, 환자 수준 불확실성, 지원 범위를 맞춘 무작위 기준선 및 기계 생성 결과를 제공합니다. @@ -13,6 +17,8 @@ NeoRepro는 공개 MHC-I 펩타이드–HLA 신생항원 예측기를 위한 데 - **현재 원고:** [리소스 중심 원고](paper/manuscript_resource.md). - **간단한 증거 요약:** [중영 이중언어 전문가 요약](output/pdf/neorepro_expert_brief_bilingual.pdf)과 [독립 코호트 확장 요약](reports/extension_summary.md). - **고정 결과 재현:** 아래 재현 절의 명령을 사용하십시오. +- **자체 예측기 벤치마크:** `neorepro benchmark predictions.csv --output-dir neorepro-results`를 실행하고 [표준 단일 파일 계약](contracts/README.md)을 참조하십시오. +- **제3자 데이터셋 또는 예측기 추가:** [plug-in contract](contracts/README.md)를 참조하십시오. - **고정 버전 인용:** [CITATION.cff](CITATION.cff), [v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1)와 [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064)를 참조하십시오. 과학적 연구 계약과 범위는 [RESEARCH_SPEC.md](RESEARCH_SPEC.md)를 참조하십시오. @@ -20,10 +26,12 @@ NeoRepro는 공개 MHC-I 펩타이드–HLA 신생항원 예측기를 위한 데 ## 상태 - 최신 문헌 감사: 완료, 결정은 `RESCOPE, then GO` -- 재현한 예측기: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN, DeepHLApan +- 벤치마크 예측기: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN, DeepHLApan. 추가로 일곱 공개 도구의 프로필 전용, 비교 불가 또는 재현 실패 기록을 버전 관리 - TESLA 파일럿: 완료, 학습 데이터 중복 양성 대조군으로 재분류 -- 주요 벤치마크: IMPROVE, 누출 필터링 후 17,475개 레코드, 70명 환자, 3개 코호트 +- 주요 벤치마크: IMPROVE, 누출 필터링 후 17,475개 레코드, 70명 환자, 세 개 코호트 - 주요 IMPROVE 추론: 완료, 고정 도구 예측 52,425개, 누락 레코드 없음 +- 외부 도메인: Zhao 백신 코호트와 별도로 고정한 129개 레코드·9명 환자의 RCC 백신 코호트 +- 재사용 가능한 확장 인터페이스: 기계 검증된 Dataset Card, Predictor Card 및 예측 아티팩트 계약 - 원고: [리소스 중심 버전](paper/manuscript_resource.md), 고정 결과 파일에서 생성, 독립 통계 및 생물학 검토 완료 ## 주요 결과 @@ -46,7 +54,7 @@ make -j4 reproduce-results - **감사 가능한 출력:** [최종 결과표](results/final_results.csv), [그림](results/figures/), [학습 데이터 중복 감사](research/training_overlap_summary_improve.json), [SHA-256 매니페스트](results/manifest.json). - **투고 계획:** [대상 저널 전략](reports/target_venues_2026-08-20.md). -독립 Zhao 2026 백신 코호트 확장은 `make -j4 extension`으로 재현할 수 있습니다. 간결한 증거 요약은 [reports/extension_summary.md](reports/extension_summary.md), 추론 전에 고정한 연구 계약은 [research/extension_protocol.json](research/extension_protocol.json)에 있습니다. 외부 종말점은 펩타이드 펄스 수지상세포 투여 후 백신 접종 뒤 ELISPOT이며, 자연 종양 제시나 임상 효능으로 해석해서는 안 됩니다. +독립 Zhao 2026 백신 코호트 확장은 `make -j4 extension`으로 재현할 수 있습니다. 간결한 증거 요약은 [reports/extension_summary.md](reports/extension_summary.md), 추론 전에 고정한 연구 계약은 [research/extension_protocol.json](research/extension_protocol.json)에 있습니다. 별도로 고정한 RCC 확장은 [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), 세 도메인 탐색적 안정성 출력은 `results/analysis/stability/`에 있습니다. 두 외부 종말점 모두 백신 접종 후 측정이며 자연 종양 제시나 임상 효능으로 해석해서는 안 됩니다. ## 라이선스 diff --git a/README.md b/README.md index fdde753..bbea1cb 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[We are actively seeking external reviewers and predictor authors.](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 15–30 min sanity checks, reproduction attempts, dataset suggestions and adversarial critiques are welcome. + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro is a leakage-aware, patient-level, reproducible benchmark resource for public MHC-I peptide–HLA neoantigen predictors. It packages pinned predictor artifacts, record-level provenance, training-overlap audits, common-support comparisons, patient-level uncertainty, support-matched random baselines and machine-generated results. @@ -13,6 +17,8 @@ This is a benchmark/resource contribution, not a new predictor and not a claim o - **Current manuscript:** [resource-positioned manuscript](paper/manuscript_resource.md). - **Two-minute evidence summary:** [bilingual expert brief](output/pdf/neorepro_expert_brief_bilingual.pdf) and [independent-cohort extension summary](reports/extension_summary.md). - **Reproduce the frozen results:** use the command in the Reproduce section below. +- **Benchmark your own predictor:** run `neorepro benchmark predictions.csv --output-dir neorepro-results`; see the [standard one-file contract](contracts/README.md). +- **Add a third-party dataset or predictor:** follow the [plug-in contract](contracts/README.md). - **Cite a fixed version:** [CITATION.cff](CITATION.cff), the [v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1) and [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064). For the scientific contract and scope, see [RESEARCH_SPEC.md](RESEARCH_SPEC.md). @@ -20,10 +26,12 @@ For the scientific contract and scope, see [RESEARCH_SPEC.md](RESEARCH_SPEC.md). ## Status - Current-literature audit: complete; decision `RESCOPE, then GO` -- Reproduced predictors: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan +- Benchmarked predictors: MHCflurry 2.2.1, BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan; seven additional public tools have versioned profile-only, non-comparable or failed reproduction records - TESLA pilot: complete; reclassified as a training-overlap-positive control - Primary benchmark: IMPROVE, 17,475 leakage-filtered rows, 70 patients, 3 cohorts - Primary IMPROVE inference: complete; 52,425 fixed-tool predictions with no missing rows +- External domains: Zhao vaccine cohort plus a separately frozen 129-record, 9-patient RCC vaccine cohort +- Reusable extension interface: machine-validated Dataset Card, Predictor Card and prediction-artifact contracts - Manuscript: [resource-positioned version](paper/manuscript_resource.md), generated from frozen result files; independent statistical and biological review complete ## Main result @@ -46,7 +54,7 @@ Independent bootstrap analyses are parallelized by Make. Use `make reproduce-res - **Auditable outputs:** [final result table](results/final_results.csv), [figures](results/figures/), [training-overlap audit](research/training_overlap_summary_improve.json), and [SHA-256 manifest](results/manifest.json). - **Submission planning:** [target-venue strategy](reports/target_venues_2026-08-20.md). -The independent Zhao 2026 vaccine-cohort extension is reproduced with `make -j4 extension`. Its concise evidence summary is in [reports/extension_summary.md](reports/extension_summary.md), with the frozen pre-inference contract in [research/extension_protocol.json](research/extension_protocol.json). The external endpoint is post-vaccination ELISPOT after peptide-pulsed dendritic-cell administration and must not be interpreted as natural tumor presentation or clinical efficacy. +The independent Zhao 2026 vaccine-cohort extension is reproduced with `make -j4 extension`. Its concise evidence summary is in [reports/extension_summary.md](reports/extension_summary.md), with the frozen pre-inference contract in [research/extension_protocol.json](research/extension_protocol.json). The separately frozen RCC extension is documented in [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json), and exploratory three-domain stability outputs are under `results/analysis/stability/`. Both external endpoints are post-vaccination assays and must not be interpreted as natural tumor presentation or clinical efficacy. ## License diff --git a/README.zh-CN.md b/README.zh-CN.md index c6742e4..9e6fdf7 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[我们正在积极寻找外部审阅者和预测器作者。](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 欢迎进行 15–30 分钟的合理性检查、复现尝试、数据集建议和对抗性批评。 + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro 是面向公开 MHC-I 肽–HLA 新抗原预测器的泄漏感知、患者级、可复现基准资源。它整合了固定版本的预测器工件、逐记录来源、训练数据重叠审计、共同支持集比较、患者级不确定性、支持集匹配的随机基线以及机器生成的结果。 @@ -13,6 +17,8 @@ NeoRepro 是面向公开 MHC-I 肽–HLA 新抗原预测器的泄漏感知、患 - **当前论文稿：**[资源定位版论文](paper/manuscript_resource.md)。 - **两分钟证据摘要：**[中英双语专家简报](output/pdf/neorepro_expert_brief_bilingual.pdf)和[独立队列扩展摘要](reports/extension_summary.md)。 - **复现冻结结果：**使用下方“复现”部分中的命令。 +- **测试你自己的预测器：**运行 `neorepro benchmark predictions.csv --output-dir neorepro-results`；参见[标准单文件契约](contracts/README.md)。 +- **接入第三方数据集或预测器：**请参阅 [plug-in contract](contracts/README.md)。 - **引用固定版本：**参见 [CITATION.cff](CITATION.cff)、[v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1) 和 [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064)。 科学研究契约与范围见 [RESEARCH_SPEC.md](RESEARCH_SPEC.md)。 @@ -20,10 +26,12 @@ NeoRepro 是面向公开 MHC-I 肽–HLA 新抗原预测器的泄漏感知、患 ## 状态 - 最新文献审计：已完成；结论为 `RESCOPE, then GO` -- 已复现预测器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 和 DeepHLApan +- 已纳入基准的预测器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 和 DeepHLApan；另有七个公开工具保留了仅档案、不可比较或复现失败的版本化记录 - TESLA 试点：已完成；重新归类为训练数据重叠阳性对照 - 主要基准：IMPROVE，17,475 条经过泄漏过滤的记录、70 名患者、3 个队列 - 主要 IMPROVE 推理：已完成；52,425 条固定工具预测，无缺失记录 +- 外部领域：Zhao 疫苗队列，以及单独冻结的 RCC 疫苗队列（129 条记录、9 名患者） +- 可复用扩展接口：经机器验证的 Dataset Card、Predictor Card 和预测工件契约 - 论文稿：[资源定位版](paper/manuscript_resource.md)，由冻结结果文件生成；独立统计学与生物学审查均已完成 ## 主要结果 @@ -46,7 +54,7 @@ Make 会并行执行相互独立的 bootstrap 分析。在 CPU 或内存受限 - **可审计输出：**[最终结果表](results/final_results.csv)、[图表](results/figures/)、[训练数据重叠审计](research/training_overlap_summary_improve.json)和 [SHA-256 清单](results/manifest.json)。 - **投稿规划：**[目标期刊策略](reports/target_venues_2026-08-20.md)。 -独立 Zhao 2026 疫苗队列扩展可通过 `make -j4 extension` 复现。简要证据摘要见 [reports/extension_summary.md](reports/extension_summary.md)，推理前冻结的研究契约见 [research/extension_protocol.json](research/extension_protocol.json)。外部终点是在肽脉冲树突状细胞给药后的疫苗接种后 ELISPOT，不应解读为自然肿瘤呈递或临床疗效。 +独立 Zhao 2026 疫苗队列扩展可通过 `make -j4 extension` 复现。简要证据摘要见 [reports/extension_summary.md](reports/extension_summary.md)，推理前冻结的研究契约见 [research/extension_protocol.json](research/extension_protocol.json)。单独冻结的 RCC 扩展记录在 [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json)，三领域探索性稳定性输出位于 `results/analysis/stability/`。两个外部终点均为疫苗接种后检测，不应解读为自然肿瘤呈递或临床疗效。 ## 许可证 diff --git a/README.zh-TW.md b/README.zh-TW.md index 9a9abf8..f0f6cd2 100644 --- a/README.zh-TW.md +++ b/README.zh-TW.md @@ -2,6 +2,10 @@ # NeoRepro +> 🧪 **[我們正積極徵求外部審閱者與預測器作者。](https://github.com/stevezkw1998/NeoRepro/issues/2)** +> +> 歡迎進行 15–30 分鐘的合理性檢查、重現嘗試、資料集建議與對抗性批評。 + [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.22037063.svg)](https://doi.org/10.5281/zenodo.22037063) NeoRepro 是一套針對公開 MHC-I 胜肽–HLA 新抗原預測器、考量資料洩漏、病人層級且可重現的基準資源。它整合固定版本的預測器產物、逐筆資料來源、訓練資料重疊稽核、共同可評估集合比較、病人層級不確定性、依支援範圍匹配的隨機基準，以及由機器產生的結果。 @@ -13,6 +17,8 @@ NeoRepro 是一套針對公開 MHC-I 胜肽–HLA 新抗原預測器、考量資 - **目前論文稿：**[資源定位版論文](paper/manuscript_resource.md)。 - **兩分鐘證據摘要：**[中英雙語專家簡報](output/pdf/neorepro_expert_brief_bilingual.pdf)與[獨立隊列延伸摘要](reports/extension_summary.md)。 - **重現凍結結果：**使用下方「重現」章節中的指令。 +- **測試你自己的預測器：**執行 `neorepro benchmark predictions.csv --output-dir neorepro-results`；請參閱[標準單一檔案契約](contracts/README.md)。 +- **接入第三方資料集或預測器：**請參閱 [plug-in contract](contracts/README.md)。 - **引用固定版本：**請參閱 [CITATION.cff](CITATION.cff)、[v0.1.1 release](https://github.com/stevezkw1998/NeoRepro/releases/tag/v0.1.1) 與 [Zenodo DOI 10.5281/zenodo.22037064](https://doi.org/10.5281/zenodo.22037064)。 科學研究契約與範圍請見 [RESEARCH_SPEC.md](RESEARCH_SPEC.md)。 @@ -20,10 +26,12 @@ NeoRepro 是一套針對公開 MHC-I 胜肽–HLA 新抗原預測器、考量資 ## 狀態 - 最新文獻稽核：已完成；決策為 `RESCOPE, then GO` -- 已重現的預測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 與 DeepHLApan +- 已納入基準的預測器：MHCflurry 2.2.1、BigMHC v1.0、PRIME 2.0、DeepImmuno-CNN 與 DeepHLApan；另有七個公開工具保留僅供描述、不可比較或重現失敗的版本化記錄 - TESLA 試驗：已完成；重新歸類為訓練資料重疊陽性對照 - 主要基準：IMPROVE，17,475 筆經資料洩漏過濾的記錄、70 位病人、3 個隊列 - 主要 IMPROVE 推論：已完成；52,425 筆固定工具預測，沒有缺漏記錄 +- 外部領域：Zhao 疫苗隊列，以及另行凍結的 RCC 疫苗隊列（129 筆記錄、9 位病人） +- 可重用延伸介面：經機器驗證的 Dataset Card、Predictor Card 與預測產物契約 - 論文稿：[資源定位版](paper/manuscript_resource.md)，由凍結結果檔案產生；獨立統計與生物學審查皆已完成 ## 主要結果 @@ -46,7 +54,7 @@ Make 會平行執行彼此獨立的 bootstrap 分析。CPU 或記憶體受限時 - **可稽核輸出：**[最終結果表](results/final_results.csv)、[圖表](results/figures/)、[訓練資料重疊稽核](research/training_overlap_summary_improve.json)與 [SHA-256 清單](results/manifest.json)。 - **投稿規劃：**[目標期刊策略](reports/target_venues_2026-08-20.md)。 -獨立 Zhao 2026 疫苗隊列延伸可用 `make -j4 extension` 重現。簡要證據摘要位於 [reports/extension_summary.md](reports/extension_summary.md)，推論前凍結的研究契約位於 [research/extension_protocol.json](research/extension_protocol.json)。外部終點是以胜肽脈衝樹突細胞施打後的疫苗接種後 ELISPOT，不應解讀為自然腫瘤呈遞或臨床療效。 +獨立 Zhao 2026 疫苗隊列延伸可用 `make -j4 extension` 重現。簡要證據摘要位於 [reports/extension_summary.md](reports/extension_summary.md)，推論前凍結的研究契約位於 [research/extension_protocol.json](research/extension_protocol.json)。另行凍結的 RCC 延伸記錄於 [research/extension_protocol_rcc_v1.json](research/extension_protocol_rcc_v1.json)，三領域探索性穩定性輸出位於 `results/analysis/stability/`。兩個外部終點均為疫苗接種後檢測，不應解讀為自然腫瘤呈遞或臨床療效。 ## 授權條款 diff --git a/contracts/README.md b/contracts/README.md new file mode 100644 index 0000000..c845311 --- /dev/null +++ b/contracts/README.md @@ -0,0 +1,44 @@ +# NeoRepro plug-in contract + +## Standard one-file entry point + +The fastest public interface accepts one UTF-8 CSV and writes both a machine-readable +`evaluation.json` and a human-readable `report.md`: + +```bash +python -m pip install . +neorepro benchmark predictions.csv --output-dir neorepro-results +``` + +Required columns: + +```text +record_id,patient_id,study_id,label,score,predictor +``` + +Each record's truth metadata must be identical across predictors. `label` is `0` or `1`, and +larger scores are better unless `score_direction` declares `lower`. Blank scores are counted as +unsupported. Optional columns are `score_direction`, `status`, `training_overlap`, `hla`, `assay` +and `cancer_type`. `training_overlap` accepts `exact`, `none` or `unknown`; exact overlaps are +excluded from the primary common-support comparison, while unknown remains unknown. + +The command reports AUROC, AUPRC, fixed-threshold classification metrics, eligible Brier score, +tie-aware patient Recall/Precision/HitRate/NDCG at K=5/10/20, MRR, patient bootstrap confidence +intervals, paired predictor differences, support-matched random ranking and available study/HLA/ +assay/cancer-type strata. Stratified views are descriptive, not held-out validation. + +See `contracts/synthetic/standard_predictions.csv` for a complete two-predictor example. + +## Card and separate-artifact interface + +Third parties can also validate cards and separate artifacts before evaluation: + +```bash +neorepro dataset validate contracts/dataset-card.example.json +neorepro predictor validate contracts/predictor-card.example.json +neorepro artifact predictions.csv --benchmark benchmark.csv +neorepro evaluate benchmark.csv predictions.csv --output results/evaluation.json --report reports/evaluation.md +neorepro overlap-audit predictions.csv +``` + +Every artifact must contain one row for every benchmark `record_id`. Failed or unsupported predictions remain explicit rows and are counted as missing; they are never imputed. `score_direction` is mandatory and evaluation orients lower-is-better scores before metrics. Unknown training overlap is reported as unknown, not as independence. diff --git a/contracts/dataset-card.example.json b/contracts/dataset-card.example.json new file mode 100644 index 0000000..97e37da --- /dev/null +++ b/contracts/dataset-card.example.json @@ -0,0 +1,10 @@ +{ + "schema_version": "dataset-card-1", + "dataset_id": "synthetic-demo", + "version": "1.0.0", + "records_path": "benchmark.csv", + "label_column": "label", + "patient_id_column": "patient_id", + "score_tasks": ["immunogenicity"], + "source": {"url": "https://example.org", "accessed": "2026-08-20", "sha256": "unknown", "license": "CC-BY-4.0"} +} diff --git a/contracts/prediction-artifact.schema.json b/contracts/prediction-artifact.schema.json new file mode 100644 index 0000000..b757730 --- /dev/null +++ b/contracts/prediction-artifact.schema.json @@ -0,0 +1,11 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "NeoRepro Prediction Artifact", + "type": "object", + "required": ["record_id", "predictor", "predictor_version", "task", "score", "score_direction", "status"], + "properties": { + "record_id": {"type": "string"}, "predictor": {"type": "string"}, "predictor_version": {"type": "string"}, + "task": {"type": "string"}, "score": {"type": ["number", "string", "null"]}, + "score_direction": {"enum": ["higher", "lower"]}, "status": {"enum": ["predicted", "unsupported", "failed", "invalid"]} + } +} diff --git a/contracts/predictor-card.example.json b/contracts/predictor-card.example.json new file mode 100644 index 0000000..79f1671 --- /dev/null +++ b/contracts/predictor-card.example.json @@ -0,0 +1,12 @@ +{ + "schema_version": "predictor-card-1", + "predictor_id": "synthetic-demo", + "version": "1.0.0", + "task": "immunogenicity", + "score_direction": "higher", + "adapter": "predictors/example/adapter.py", + "license": "MIT", + "input": ["record_id", "peptide", "hla"], + "output": "contracts/prediction-artifact.schema.json", + "training_overlap": "unknown" +} diff --git a/contracts/synthetic/README.md b/contracts/synthetic/README.md new file mode 100644 index 0000000..9043a4c --- /dev/null +++ b/contracts/synthetic/README.md @@ -0,0 +1,10 @@ +# Synthetic plug-in example + +This four-row fixture demonstrates explicit unsupported output, common-support evaluation, and patient-level ranking without using biological data. + +Run the portable standard entry point with: + +```bash +neorepro benchmark contracts/synthetic/standard_predictions.csv \ + --output-dir neorepro-results --bootstrap 100 +``` diff --git a/contracts/synthetic/benchmark.csv b/contracts/synthetic/benchmark.csv new file mode 100644 index 0000000..b776520 --- /dev/null +++ b/contracts/synthetic/benchmark.csv @@ -0,0 +1,5 @@ +record_id,patient_id,peptide,hla,label +s1,P1,SIINFEKL,HLA-A*02:01,1 +s2,P1,NLVPMVATV,HLA-A*02:01,0 +s3,P2,GLCTLVAML,HLA-A*02:01,1 +s4,P2,LLD, HLA-A*02:01,0 diff --git a/contracts/synthetic/predictions.csv b/contracts/synthetic/predictions.csv new file mode 100644 index 0000000..1dd4095 --- /dev/null +++ b/contracts/synthetic/predictions.csv @@ -0,0 +1,5 @@ +record_id,predictor,predictor_version,task,score,score_direction,status +s1,synthetic,1.0,immunogenicity,0.9,higher,predicted +s2,synthetic,1.0,immunogenicity,0.1,higher,predicted +s3,synthetic,1.0,immunogenicity,0.8,higher,predicted +s4,synthetic,1.0,immunogenicity,,higher,unsupported diff --git a/contracts/synthetic/standard_predictions.csv b/contracts/synthetic/standard_predictions.csv new file mode 100644 index 0000000..fb76a73 --- /dev/null +++ b/contracts/synthetic/standard_predictions.csv @@ -0,0 +1,9 @@ +record_id,patient_id,study_id,label,score,predictor,score_direction,status,training_overlap,hla +s1,P1,SYNTHETIC,1,0.9,Example-A,higher,predicted,none,HLA-A*02:01 +s2,P1,SYNTHETIC,0,0.1,Example-A,higher,predicted,none,HLA-A*02:01 +s3,P2,SYNTHETIC,1,0.8,Example-A,higher,predicted,unknown,HLA-A*02:01 +s4,P2,SYNTHETIC,0,0.2,Example-A,higher,predicted,none,HLA-A*02:01 +s1,P1,SYNTHETIC,1,0.7,Example-B,higher,predicted,none,HLA-A*02:01 +s2,P1,SYNTHETIC,0,0.3,Example-B,higher,predicted,none,HLA-A*02:01 +s3,P2,SYNTHETIC,1,0.6,Example-B,higher,predicted,unknown,HLA-A*02:01 +s4,P2,SYNTHETIC,0,,Example-B,higher,unsupported,none,HLA-A*02:01 diff --git a/data/predictor_registry.csv b/data/predictor_registry.csv index 94f9ab9..e25085f 100644 --- a/data/predictor_registry.csv +++ b/data/predictor_registry.csv @@ -4,3 +4,10 @@ BigMHC,v1.0,9d84a3b4da77c9253ac90ff8cb629274003b90fd,BigMHC-Academic-License,yes PRIME,2.0,ec1aa020089d62e9193ad377ddda9c93eed7f5b1,LICR-academic-noncommercial,yes,no,yes,yes,yes,complete,8.50,yes,none,no,no,no,macOS-15.3-arm64,no,no,yes,reproduced,predictors/prime/attempts/,"17,475/17,475 IMPROVE predictions plus the 520-row leakage-positive fixture; requires MixMHCpred v2.2 at f64bb4548082768c70a1cfb5a4442d5e6ea04591; both C++ binaries recompiled for ARM64; 146-row official fixture matched exactly; upstream artifacts are not redistributed." DeepImmuno-CNN,1.0,df42ac5b6bddfe531268335e2dcb496559cd488b,MIT,no,yes,yes,yes,yes,complete,unknown,yes,none,no,no,no,macOS-15.3-arm64,no,no,yes,reproduced,predictors/deepimmuno/,"Published checkpoint loaded under isolated TensorFlow 2.15.1 environment; exact HLA only, no upstream fuzzy rescue; 9–10mer contract; public 9,055-row training table audited for exact overlap." DeepHLApan,1.1.1,ac1f4bebc095271504dfc2d2a93888df3be94e83,GPL-2.0,no,yes,yes,yes,yes,complete,unknown,yes,legacy_Keras_GRU_loader,no,no,no,macOS-15.3-arm64,no,no,yes,reproduced,predictors/deephlapan/,"All five published binding and immunogenicity models load through a reset_after=False compatibility shim and are averaged; exact HLA only. Official row-level training manifest unavailable, so overlap is unknown." +MHCnuggets,2.4.0,b666fea3a54a1d357efba4ea4d8550ce5dd50aba,unclear,unknown,unknown,yes,yes,yes,complete,7.53,no,none,no,no,no,macOS-15.3-arm64,optional,no,yes,pending,predictors/mhcnuggets/attempts/,"Production BA weights are vendored in the pinned source; official test peptide fixture ran through the upstream API and strict adapter (3/3). Lower IC50 is better. The repository's separate saves/test model has different expected values, so exact test-model comparison is not claimed. Binding-track comparability and training-overlap audit remain pending." +NeoGuider,unknown,f95a5713453910902d657f1d148568b93c81026f,AGPL-3.0,unknown,yes,yes,no,no,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,unknown,yes,unknown,attempted_unreproducible,predictors/neoguider/attempts/,"Repository lacks pyproject.toml/setup.py; documented route is shell/conda. No benchmark entry." +NeoFox,1.2.3,dd894a67ad02a4377ed3e4b1e44bd576360c98cd,GPL-3.0,no,yes,yes,yes,yes,none,unknown,no,none,no,no,unknown,macOS-15.3-arm64,no,optional,yes,pending,predictors/neofox/attempts/,"Editable install and import smoke test passed; annotation output is not peptide-HLA score comparable. Profile-only pending official fixture and service audit." +pVACtools,7.1.2,2ce871be5ae56af141fe723d60d3841125874a74,BSD-3-Clause-Clear,no,yes,yes,yes,yes,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,no,optional,yes,excluded_noncomparable,predictors/pvactools/attempts/,"Editable install and pvacseq --help smoke test passed; end-to-end orchestration, not a single comparable score model." +Seq2Neo,2.1,f342aa4ffa9012fb5509914333b9401a1eb286c4,AFL-3.0,unknown,yes,yes,no,no,none,unknown,no,dependency_conflict:no_tensorflow_2.3_cp311,no,no,unknown,macOS-15.3-arm64,unknown,yes,unknown,attempted_unreproducible,predictors/seq2neo/attempts/,"Install failed under Python 3.11 because tensorflow==2.3.0 has no cp311 wheel; legacy route remains follow-up. Profile-only." +Vaxrank,unknown,5c71edccda920fafaab5986837d3dd87c8ce8911,Apache-2.0,no,yes,yes,yes,yes,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,no,optional,yes,excluded_noncomparable,predictors/vaxrank/attempts/,"Editable install and help smoke test passed; vaccine-construct ranking is not peptide-HLA comparable." +mhcmatch,0.25.0,956bdb613fdb1e128cbd7d59fabb9b47ef1aaae9,GPL-3.0,no,yes,yes,yes,yes,none,unknown,no,none,no,unknown,unknown,macOS-15.3-arm64,no,no,unknown,pending,predictors/mhcmatch/attempts/,"Editable install, --help, and local decompose smoke tests passed. Binder score timed out after 180s while triggering calibration/reference bootstrap; preserved as a resource/data dependency result. Exploratory only pending method paper and training provenance." diff --git a/data/processed/rcc_vaccine_benchmark.csv b/data/processed/rcc_vaccine_benchmark.csv new file mode 100644 index 0000000..54540f3 --- /dev/null +++ b/data/processed/rcc_vaccine_benchmark.csv @@ -0,0 +1,130 @@ +record_id,patient_id,study_id,hla,mhc_class,peptide,peptide_length,immunogenicity,label,assay_type,clinical_context,source_doi,source_url,source_checksum,source_row,source_pvalue,evidence_level +rcc-vaccine-8c2e1ec6d8f675cf,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,2,5.6006335776299125E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6a6ffb8b81ec9208,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,KEKIREYEQAL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,3,4.4603377394004801E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4adeedbe0e35cf09,RCC-101,RCC_PCV_VACCINE_2025,HLA-C*06:02;HLA-A*66:01,I,VRNSRTSGY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,4,8.8070660481353924E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e06799bdf8de8af5,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,5,3.7959293105110106E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-969998ab9d8b4dc1,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,SVASHGGHQPY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,6,0.89810982568718123,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a5952e9e71bd27bd,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,GEILLPRDV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,7,0.26875586775158017,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6fa3e9ad450e030e,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,8,0.73956419203215784,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8a9041cc55b08e38,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,TETDSDPMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,9,4.3972842081206977E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e11c81feedc5751e,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,LFRGLQSR,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,10,1.1492768296259299E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fefffdcccca8649c,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,REGTGAVGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,11,5.6966328761325709E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-940b62df9c3078c4,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,ITLPVYSER,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,12,2.1248320025947062E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-43742d14e082c2e6,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,RAAASSRRPS,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,13,5.828028176445943E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-769c6b305827c570,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,FPSETNLSKEV,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,14,0.10474802857947568,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e74f183589095f61,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,NAFVEEVM,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,15,0.17473444899797558,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-15ab70cb934aad8f,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,FHFVIYVPI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,16,5.0774964296645174E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c8a8ce5ab15f9d4b,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,LALPLQRTW,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,17,0.92524545777236522,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f05602883865f4bf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,LPFDLKNPF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,18,0.80801446661256293,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b92105a12fb6f028,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,AHYSTDTAL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,19,0.47154937914914336,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a004075aed453bcf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:01,I,LPSMQEDL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,20,0.3095552136678249,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70eb710f90e2973b,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,EPAQFPEPNSF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,21,3.6050424739501194E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c81528933be4a147,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,FQESYFESY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,22,9.2338784047006368E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-2a2f0b8f959aa3f6,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,HPPPHIQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,23,0.21566601684775977,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-74e8a99b7ef86c61,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LLYNKMIL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,24,1.2335170096117602E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88bea2527dd72a1d,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,GSPAITHRNL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,25,2.3339160452827509E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ff6bbb8d69468a47,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,SRNDVSRL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,26,2.5026099459392787E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8ea6c4f21d2c81ee,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ILLGEDDSVQI,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,27,0.18350341907227408,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2f6110d808ac0da,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,IVGPLPPPL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,28,0.2254033307585166,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f43e93564087f160,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,FRPDSKSSAL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,29,3.5098718645984656E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fc1276f53c3f45e1,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,KKGRPGQTGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,30,0.19930188719405426,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3e6b1c1d8de0ce71,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,VAFDFTKVE,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,31,0.47311639825056773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4255bdded9e984bd,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,SRGAAGWSL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,32,0.57898632350489265,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-31c3a5a1c962db43,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LPRPKIARL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,33,8.3377373835244337E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d3d4046d155047b6,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ALSEQIQMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,34,1.4246046683842653E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5e4096b57e93c92e,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,LLKLLHHRY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,35,2.0118966840664707E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e8d6aeb51705642a,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,ARWGETVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,36,1.8306664737796626E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-26dc24e186221fe2,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,GTKPLYVVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,37,5.0221254228049662E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-364822c022710958,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,ILKRRTNM,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,38,4.7676050558719381E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9c477d13a3359f5e,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,39,7.0908120808195439E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-487a8e43d12c17ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,EILERRFL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,40,0.13919156951235334,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-539b50b223ea0ee1,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,41,2.3027081004049618E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-618d6aa7d8232657,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,42,8.1766449056166948E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-71f5d2ab19a389e8,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,STATLPFHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,43,1.846734671054058E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-409a0661a319bbb1,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,44,6.6867747669650493E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9f431d040990f0f7,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,YSNWPDKLY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,45,8.0583222939814639E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7c1009a0bb0524b3,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,AFFDGKTL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,46,4.8824113766337794E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-42e869aba90dc4bd,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ISHISPQRGFH,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,47,1.7166906883565791E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a1a88d1c11b4fcdd,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,GGGAEKVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,48,0.42264973081037416,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d42b9fb3a44366ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,49,1.021179201718677E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ce806d8b1a4132b2,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,50,2.3380484295577471E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ea51c18cc4e16995,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,51,1.7222634968187069E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9bce007ba3a4aff0,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,STGLRVIK,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,52,1.0052468182969208E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa3019acbdddb7dc,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,YGCPHLQQ,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,53,0.16109342700158688,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1bcdaed331509a08,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,54,5.9493457814588473E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4a1853322eea7a14,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,MRILHNKGI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,55,2.131276725678492E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-cde49959d75e6ec2,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FMNSSTSL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,56,1.773461347691256E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7f8e1f8e02332663,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ESQKEMKLLLY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,57,0.67173725533059248,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8aa4d97d214f765,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,MSIPGMEEQL,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,58,6.0793325180986964E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5993797f7a6c7057,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,RLRDDVMRLSR,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,59,1.400860297501039E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-eacb2f310488ca9d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,VEAFDADMLKL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,60,1.214858610163266E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70f05fcfa038224f,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,HSPKYPSPYR,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,61,0.67085616631628397,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d15c4e62bb19311,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,PPKSLFQL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,62,4.6693166687862796E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a6b2180415ff1b56,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,KPEIAHVTSA,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,63,4.2382661368441126E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e52f3609d694b4ba,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KLDTGFKILDR,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,64,0.27589910187209943,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-763f29ff7ac4c65d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,METIEKIMGDL,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,65,0.76746069478425916,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c331f65899396b9b,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,YVISRTSPV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,66,8.0195333258099456E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-09c9b699d977f421,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,KLWLLRPLR,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,67,7.1646552252994013E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e944d5a782a7d074,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,LDAPDFQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,68,0.51383607553050725,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-75f86cf28dca60e2,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,IINEASQVKPK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,69,7.2477924894824214E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5b2ef450eaf2248a,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,FAFYGAHGAQF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,70,6.5233637469130547E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c50e7f0e1b3b69a5,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KAKPVIFLGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,71,7.0798177776713822E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ae70f68909b113e8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,DEITATYML,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,72,7.0303170829206542E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-efdf21635c142ab8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,SPYHYQRV,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,73,5.6759276880420054E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-817b1ac6328997a7,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KIMKKAIERGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,74,1.6379409412884873E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-feb66d3c66c8a4de,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,GLWELPVEGRA,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,75,0.19170962313452389,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7946e9a10fe114cf,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LDMKGEIAEAY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,76,4.8810268788658163E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f3e32e857dd09d71,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,AENNDLLWMNY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,77,0.43819304962341288,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-240bf5b232815c6a,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VIEPHDILV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,78,1.6539295298740098E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-40c01a17a3ed4a2e,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,KLYEAEDDFKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,79,5.025231714396812E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7b823cc728ea1450,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,RILDAREHSY,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,80,0.23019964108049898,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0c7eb5e3295f0ad2,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,REVRLQNKTSY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,81,3.7355310355967439E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88aaf2311ce63196,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,ASDGLSERPL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,82,8.1513907431533788E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-bcae618cfad77c1f,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,PDMVVSHLFW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,83,0.56115088124008583,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-11f61d47947266cc,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KLQQPDHASW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,84,2.2184135540696064E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f4a4db2362e7a54e,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,86,3.774955135062371E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1eceb5b6a969d721,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,SRKGSIKGLVY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,87,7.417990022744847E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-55106745d24cfbf4,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,SLNFKNPEA,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,88,1.3963573524385535E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a808947689fbe42c,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,FRFTPPNPEA,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,89,0.11808289631180313,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-590f5ea08f24e2cf,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,PRHAAGGLHF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,90,5.9124927719229947E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b984dc3be5c1fa17,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,91,8.2441737161747519E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6ff6e758ac279425,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YAVPGPIVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,92,6.3803433637722395E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa0d4e0503dacba4,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,WRELNDLGSK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,93,0.43261607750026987,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d44829f31cf5558,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRPETFNHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,94,2.9913865953409766E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4f7d9f9f22d48440,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRLSHFEYVKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,95,0.7467841772335857,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-60d1ba1b8ae350fd,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,96,1.281550265552316E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7ec85836cb09ee30,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:06,I,HQYVPKDQRL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,97,1.4725369350184756E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4b7fdcc54f65b98b,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,98,9.0220118727937296E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d02de725ecc685a5,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARTFDKKRF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,99,5.8042374560036136E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e1d984ebf245728f,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,LRSILPSTGI,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,100,3.5960591985765922E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a9a3c2bc98a74746,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,101,4.5821799894541555E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-da610226e338a8de,RCC-109,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YLMDISGKV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,102,2.448656045672399E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ca0a8a8bb4e2d094,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,AAGSASSSKV,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,103,4.0408082142463435E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c45585f253341070,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,104,4.5974381758390748E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2761351ccb757b7,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,105,0.29539930932276304,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b1ad936ccc2f249a,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,106,5.9503855187285459E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-02261377bb800592,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,107,4.7874694511645952E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-834aff6d2a7dc363,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,108,2.2477796563320426E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0036b8c45a4cd20d,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,109,0.37853076038781203,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-08447808dd0300dd,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,110,6.4833740564642257E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-34c3e7e7a2dcd8b6,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,111,6.7901063097408093E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1ae968c5ae01f49b,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,112,2.1954978277654386E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-29f7c92d094c1ced,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,EEVLEIAASGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,113,2.2356452288465851E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0aaef5904890371c,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,114,2.8827717670021147E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-27b47d43dab2b353,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VTDTFSDV,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,115,0.24291648886267675,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-46b486b9d199e93a,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,ARGPMPVHTT,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,116,6.2291361519685288E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8a9c44bcf53568f,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KVLKLIPEK,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,117,4.7559190547222883E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7425139a6230adc0,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,118,1.1997646355098002E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e39c49c350f42009,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,RPSQPSRERL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,119,4.6026306166334421E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-73717d4619b66faf,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,MREIREVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,120,1,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f130cbb5ea1ca0fa,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,HSLEMMNSM,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,121,1.9687903060187317E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3f16099c911d2791,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,122,0.28825566505998773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-07bc8054e1edd65a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,VPNHLPPAL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,123,2.378495810276827E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-99bd603a759d6b22,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,RYSIHSEL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,124,3.0348245596030862E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ad726d8ef91bd0e6,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,SRGDGEHF,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,125,6.6703592263578429E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-972e1f28fe4cb96a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEQNATKSW,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,126,1.6695522924906413E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e0d628668c1eccf2,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*25:01,I,DIEDGEADHI,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,127,9.8999275264303174E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3d165d772065bb3a,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,YAFSPPQGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,128,3.3849045480144387E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9ef49682993d2ea1,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,PPAPQPQRL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,129,0.65512239646493831,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6c01a27a6a575007,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,ARWHRAQDL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,130,1.266837038557146E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-12b152d35907e280,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,KPPKAYSVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,131,0.19485075298185836,individual_peptide_stimulation_with_matched_no_stimulation_control diff --git a/data/processed/rcc_vaccine_benchmark_full.csv b/data/processed/rcc_vaccine_benchmark_full.csv new file mode 100644 index 0000000..54540f3 --- /dev/null +++ b/data/processed/rcc_vaccine_benchmark_full.csv @@ -0,0 +1,130 @@ +record_id,patient_id,study_id,hla,mhc_class,peptide,peptide_length,immunogenicity,label,assay_type,clinical_context,source_doi,source_url,source_checksum,source_row,source_pvalue,evidence_level +rcc-vaccine-8c2e1ec6d8f675cf,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,2,5.6006335776299125E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6a6ffb8b81ec9208,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,KEKIREYEQAL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,3,4.4603377394004801E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4adeedbe0e35cf09,RCC-101,RCC_PCV_VACCINE_2025,HLA-C*06:02;HLA-A*66:01,I,VRNSRTSGY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,4,8.8070660481353924E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e06799bdf8de8af5,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,5,3.7959293105110106E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-969998ab9d8b4dc1,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,SVASHGGHQPY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,6,0.89810982568718123,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a5952e9e71bd27bd,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,GEILLPRDV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,7,0.26875586775158017,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6fa3e9ad450e030e,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*57:01,I,ITKQEKDFLW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,8,0.73956419203215784,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8a9041cc55b08e38,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,TETDSDPMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,9,4.3972842081206977E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e11c81feedc5751e,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,LFRGLQSR,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,10,1.1492768296259299E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fefffdcccca8649c,RCC-101,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,REGTGAVGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,11,5.6966328761325709E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-940b62df9c3078c4,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,ITLPVYSER,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,12,2.1248320025947062E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-43742d14e082c2e6,RCC-101,RCC_PCV_VACCINE_2025,HLA-A*66:01,I,RAAASSRRPS,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,13,5.828028176445943E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-769c6b305827c570,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,FPSETNLSKEV,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,14,0.10474802857947568,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e74f183589095f61,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,NAFVEEVM,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,15,0.17473444899797558,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-15ab70cb934aad8f,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,FHFVIYVPI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,16,5.0774964296645174E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c8a8ce5ab15f9d4b,RCC-102,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,LALPLQRTW,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,17,0.92524545777236522,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f05602883865f4bf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,LPFDLKNPF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,18,0.80801446661256293,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b92105a12fb6f028,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*38:01,I,AHYSTDTAL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,19,0.47154937914914336,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a004075aed453bcf,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:01,I,LPSMQEDL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,20,0.3095552136678249,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70eb710f90e2973b,RCC-102,RCC_PCV_VACCINE_2025,HLA-B*35:03,I,EPAQFPEPNSF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,21,3.6050424739501194E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c81528933be4a147,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,FQESYFESY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,22,9.2338784047006368E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-2a2f0b8f959aa3f6,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,HPPPHIQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,23,0.21566601684775977,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-74e8a99b7ef86c61,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LLYNKMIL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,24,1.2335170096117602E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88bea2527dd72a1d,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,GSPAITHRNL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,25,2.3339160452827509E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ff6bbb8d69468a47,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,SRNDVSRL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,26,2.5026099459392787E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-8ea6c4f21d2c81ee,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ILLGEDDSVQI,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,27,0.18350341907227408,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2f6110d808ac0da,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,IVGPLPPPL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,28,0.2254033307585166,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f43e93564087f160,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,FRPDSKSSAL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,29,3.5098718645984656E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-fc1276f53c3f45e1,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,KKGRPGQTGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,30,0.19930188719405426,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3e6b1c1d8de0ce71,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,VAFDFTKVE,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,31,0.47311639825056773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4255bdded9e984bd,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,SRGAAGWSL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,32,0.57898632350489265,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-31c3a5a1c962db43,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,LPRPKIARL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,33,8.3377373835244337E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d3d4046d155047b6,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*13:02,I,ALSEQIQMV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,34,1.4246046683842653E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5e4096b57e93c92e,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,LLKLLHHRY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,35,2.0118966840664707E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e8d6aeb51705642a,RCC-104,RCC_PCV_VACCINE_2025,HLA-C*06:02,I,ARWGETVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,36,1.8306664737796626E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-26dc24e186221fe2,RCC-104,RCC_PCV_VACCINE_2025,HLA-A*30:01,I,GTKPLYVVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,37,5.0221254228049662E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-364822c022710958,RCC-104,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,ILKRRTNM,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,38,4.7676050558719381E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9c477d13a3359f5e,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,39,7.0908120808195439E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-487a8e43d12c17ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,EILERRFL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,40,0.13919156951235334,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-539b50b223ea0ee1,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,41,2.3027081004049618E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-618d6aa7d8232657,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,42,8.1766449056166948E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-71f5d2ab19a389e8,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,STATLPFHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,43,1.846734671054058E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-409a0661a319bbb1,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,PRSDNEKWNF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,44,6.6867747669650493E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9f431d040990f0f7,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,YSNWPDKLY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,45,8.0583222939814639E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7c1009a0bb0524b3,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,AFFDGKTL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,46,4.8824113766337794E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-42e869aba90dc4bd,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ISHISPQRGFH,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,47,1.7166906883565791E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a1a88d1c11b4fcdd,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,GGGAEKVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,48,0.42264973081037416,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d42b9fb3a44366ac,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,VQDRMGSGLLY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,49,1.021179201718677E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ce806d8b1a4132b2,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,50,2.3380484295577471E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ea51c18cc4e16995,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,RRSSSWRTSL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,51,1.7222634968187069E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9bce007ba3a4aff0,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,STGLRVIK,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,52,1.0052468182969208E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa3019acbdddb7dc,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,YGCPHLQQ,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,53,0.16109342700158688,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1bcdaed331509a08,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,KTERLFSYMY,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,54,5.9493457814588473E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4a1853322eea7a14,RCC-105,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,MRILHNKGI,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,55,2.131276725678492E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-cde49959d75e6ec2,RCC-105,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FMNSSTSL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,56,1.773461347691256E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7f8e1f8e02332663,RCC-105,RCC_PCV_VACCINE_2025,HLA-A*01:01,I,ESQKEMKLLLY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,57,0.67173725533059248,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8aa4d97d214f765,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,MSIPGMEEQL,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,58,6.0793325180986964E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5993797f7a6c7057,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,RLRDDVMRLSR,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,59,1.400860297501039E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-eacb2f310488ca9d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,VEAFDADMLKL,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,60,1.214858610163266E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-70f05fcfa038224f,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,HSPKYPSPYR,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,61,0.67085616631628397,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d15c4e62bb19311,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,PPKSLFQL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,62,4.6693166687862796E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a6b2180415ff1b56,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,KPEIAHVTSA,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,63,4.2382661368441126E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e52f3609d694b4ba,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KLDTGFKILDR,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,64,0.27589910187209943,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-763f29ff7ac4c65d,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,METIEKIMGDL,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,65,0.76746069478425916,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c331f65899396b9b,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,YVISRTSPV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,66,8.0195333258099456E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-09c9b699d977f421,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*31:01,I,KLWLLRPLR,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,67,7.1646552252994013E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e944d5a782a7d074,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,LDAPDFQL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,68,0.51383607553050725,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-75f86cf28dca60e2,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,IINEASQVKPK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,69,7.2477924894824214E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-5b2ef450eaf2248a,RCC-106,RCC_PCV_VACCINE_2025,HLA-C*03:03,I,FAFYGAHGAQF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,70,6.5233637469130547E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c50e7f0e1b3b69a5,RCC-106,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KAKPVIFLGK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,71,7.0798177776713822E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ae70f68909b113e8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*40:01,I,DEITATYML,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,72,7.0303170829206542E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-efdf21635c142ab8,RCC-106,RCC_PCV_VACCINE_2025,HLA-B*55:01,I,SPYHYQRV,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,73,5.6759276880420054E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-817b1ac6328997a7,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KIMKKAIERGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,74,1.6379409412884873E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-feb66d3c66c8a4de,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,GLWELPVEGRA,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,75,0.19170962313452389,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7946e9a10fe114cf,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LDMKGEIAEAY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,76,4.8810268788658163E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f3e32e857dd09d71,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,AENNDLLWMNY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,77,0.43819304962341288,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-240bf5b232815c6a,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VIEPHDILV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,78,1.6539295298740098E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-40c01a17a3ed4a2e,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,KLYEAEDDFKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,79,5.025231714396812E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7b823cc728ea1450,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,RILDAREHSY,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,80,0.23019964108049898,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0c7eb5e3295f0ad2,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,REVRLQNKTSY,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,81,3.7355310355967439E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-88aaf2311ce63196,RCC-107,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,ASDGLSERPL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,82,8.1513907431533788E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-bcae618cfad77c1f,RCC-107,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,PDMVVSHLFW,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,83,0.56115088124008583,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-11f61d47947266cc,RCC-107,RCC_PCV_VACCINE_2025,HLA-A*32:01,I,KLQQPDHASW,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,84,2.2184135540696064E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f4a4db2362e7a54e,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,86,3.774955135062371E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1eceb5b6a969d721,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,SRKGSIKGLVY,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,87,7.417990022744847E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-55106745d24cfbf4,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,SLNFKNPEA,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,88,1.3963573524385535E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a808947689fbe42c,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,FRFTPPNPEA,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,89,0.11808289631180313,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-590f5ea08f24e2cf,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,PRHAAGGLHF,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,90,5.9124927719229947E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b984dc3be5c1fa17,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,ALYSPLMSV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,91,8.2441737161747519E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6ff6e758ac279425,RCC-108,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YAVPGPIVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,92,6.3803433637722395E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-aa0d4e0503dacba4,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,WRELNDLGSK,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,93,0.43261607750026987,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4d44829f31cf5558,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRPETFNHL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,94,2.9913865953409766E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4f7d9f9f22d48440,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,TRLSHFEYVKK,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,95,0.7467841772335857,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-60d1ba1b8ae350fd,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,96,1.281550265552316E-5,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7ec85836cb09ee30,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:06,I,HQYVPKDQRL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,97,1.4725369350184756E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-4b7fdcc54f65b98b,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARVAQRLKL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,98,9.0220118727937296E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-d02de725ecc685a5,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,ARTFDKKRF,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,99,5.8042374560036136E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e1d984ebf245728f,RCC-108,RCC_PCV_VACCINE_2025,HLA-B*27:05,I,LRSILPSTGI,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,100,3.5960591985765922E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-a9a3c2bc98a74746,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,101,4.5821799894541555E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-da610226e338a8de,RCC-109,RCC_PCV_VACCINE_2025,HLA-A*02:01,I,YLMDISGKV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,102,2.448656045672399E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ca0a8a8bb4e2d094,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,AAGSASSSKV,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,103,4.0408082142463435E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-c45585f253341070,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,104,4.5974381758390748E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b2761351ccb757b7,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,105,0.29539930932276304,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b1ad936ccc2f249a,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,106,5.9503855187285459E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-02261377bb800592,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,107,4.7874694511645952E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-834aff6d2a7dc363,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,108,2.2477796563320426E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0036b8c45a4cd20d,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,KEALPPVLL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,109,0.37853076038781203,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-08447808dd0300dd,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*08:01,I,FLKKKFLRV,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,110,6.4833740564642257E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-34c3e7e7a2dcd8b6,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,LEFEKQRNNSW,11,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,111,6.7901063097408093E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-1ae968c5ae01f49b,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,112,2.1954978277654386E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-29f7c92d094c1ced,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,EEVLEIAASGF,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,113,2.2356452288465851E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-0aaef5904890371c,RCC-109,RCC_PCV_VACCINE_2025,HLA-B*44:02,I,QMVHPVAERLW,11,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,114,2.8827717670021147E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-27b47d43dab2b353,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*05:01,I,VTDTFSDV,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,115,0.24291648886267675,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-46b486b9d199e93a,RCC-109,RCC_PCV_VACCINE_2025,HLA-C*07:01,I,ARGPMPVHTT,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,116,6.2291361519685288E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-b8a9c44bcf53568f,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*03:01,I,KVLKLIPEK,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,117,4.7559190547222883E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-7425139a6230adc0,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,118,1.1997646355098002E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e39c49c350f42009,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,RPSQPSRERL,10,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,119,4.6026306166334421E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-73717d4619b66faf,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,MREIREVL,8,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,120,1,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-f130cbb5ea1ca0fa,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,HSLEMMNSM,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,121,1.9687903060187317E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3f16099c911d2791,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEPKTFQAY,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,122,0.28825566505998773,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-07bc8054e1edd65a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,VPNHLPPAL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,123,2.378495810276827E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-99bd603a759d6b22,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,RYSIHSEL,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,124,3.0348245596030862E-4,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-ad726d8ef91bd0e6,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,SRGDGEHF,8,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,125,6.6703592263578429E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-972e1f28fe4cb96a,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*18:01,I,SEQNATKSW,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,126,1.6695522924906413E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-e0d628668c1eccf2,RCC-110,RCC_PCV_VACCINE_2025,HLA-A*25:01,I,DIEDGEADHI,10,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,127,9.8999275264303174E-2,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-3d165d772065bb3a,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*12:03,I,YAFSPPQGL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,128,3.3849045480144387E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-9ef49682993d2ea1,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,PPAPQPQRL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,129,0.65512239646493831,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-6c01a27a6a575007,RCC-110,RCC_PCV_VACCINE_2025,HLA-C*07:02,I,ARWHRAQDL,9,1,1,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,130,1.266837038557146E-3,individual_peptide_stimulation_with_matched_no_stimulation_control +rcc-vaccine-12b152d35907e280,RCC-110,RCC_PCV_VACCINE_2025,HLA-B*07:02,I,KPPKAYSVL,9,0,0,post_vaccine_invitro_IFNG_ELISPOT,personalized_RCC_peptide_vaccine,10.1038/s41586-024-08507-5,https://doi.org/10.1038/s41586-024-08507-5,c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1,131,0.19485075298185836,individual_peptide_stimulation_with_matched_no_stimulation_control diff --git a/data/rcc_vaccine_summary.json b/data/rcc_vaccine_summary.json new file mode 100644 index 0000000..8d71fa5 --- /dev/null +++ b/data/rcc_vaccine_summary.json @@ -0,0 +1,9 @@ +{ + "rows": 129, + "patients": 9, + "positives": 75, + "negatives": 54, + "member_sha256": "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1", + "endpoint": "post-vaccine individual-peptide IFNG ELISpot", + "hla_semantics": "predicted best short-epitope binding allele" +} diff --git a/i18n/readme_state.json b/i18n/readme_state.json index 80aa1e6..64fe5a4 100644 --- a/i18n/readme_state.json +++ b/i18n/readme_state.json @@ -3,89 +3,89 @@ "de": { "file": "README.de.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "b7cfbdccff4b2102e3394006c7bf8d47522397d055b3e8631e481a9db511d123", + "preamble": "65fac58b5272dd01e2452877dd1c25c81c390fb0eb3dc31389c0404b905cca31", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "661bcd24c5cef092cf4d995ca102f740ae62d0ac5b35c1383b421f101cc8ba50", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "58be1f12106bbb1d720d7268f1b316aa9e2658c3112b4b9cb50afcb54635467c", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "es": { "file": "README.es.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "b7cfbdccff4b2102e3394006c7bf8d47522397d055b3e8631e481a9db511d123", + "preamble": "65fac58b5272dd01e2452877dd1c25c81c390fb0eb3dc31389c0404b905cca31", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "661bcd24c5cef092cf4d995ca102f740ae62d0ac5b35c1383b421f101cc8ba50", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "58be1f12106bbb1d720d7268f1b316aa9e2658c3112b4b9cb50afcb54635467c", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "fr": { "file": "README.fr.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "b7cfbdccff4b2102e3394006c7bf8d47522397d055b3e8631e481a9db511d123", + "preamble": "65fac58b5272dd01e2452877dd1c25c81c390fb0eb3dc31389c0404b905cca31", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "661bcd24c5cef092cf4d995ca102f740ae62d0ac5b35c1383b421f101cc8ba50", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "58be1f12106bbb1d720d7268f1b316aa9e2658c3112b4b9cb50afcb54635467c", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "ja": { "file": "README.ja.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "b7cfbdccff4b2102e3394006c7bf8d47522397d055b3e8631e481a9db511d123", + "preamble": "65fac58b5272dd01e2452877dd1c25c81c390fb0eb3dc31389c0404b905cca31", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "661bcd24c5cef092cf4d995ca102f740ae62d0ac5b35c1383b421f101cc8ba50", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "58be1f12106bbb1d720d7268f1b316aa9e2658c3112b4b9cb50afcb54635467c", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "ko": { "file": "README.ko.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "b7cfbdccff4b2102e3394006c7bf8d47522397d055b3e8631e481a9db511d123", + "preamble": "65fac58b5272dd01e2452877dd1c25c81c390fb0eb3dc31389c0404b905cca31", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "661bcd24c5cef092cf4d995ca102f740ae62d0ac5b35c1383b421f101cc8ba50", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "58be1f12106bbb1d720d7268f1b316aa9e2658c3112b4b9cb50afcb54635467c", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "zh-CN": { "file": "README.zh-CN.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "b7cfbdccff4b2102e3394006c7bf8d47522397d055b3e8631e481a9db511d123", + "preamble": "65fac58b5272dd01e2452877dd1c25c81c390fb0eb3dc31389c0404b905cca31", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "661bcd24c5cef092cf4d995ca102f740ae62d0ac5b35c1383b421f101cc8ba50", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "58be1f12106bbb1d720d7268f1b316aa9e2658c3112b4b9cb50afcb54635467c", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } }, "zh-TW": { "file": "README.zh-TW.md", "sections": { - "evidence-trail": "d59dadcd64bcd6fb91d6b2f0e17c2bcbbdda83f5ffb0e706dde0d3254f730f12", + "evidence-trail": "4cdc09f95083640e5991141c7dd0a2881b7abaa14be8bba5cdaa3e7118781fc9", "license": "c8b5a3d845f447f3d200d9eccc802340c77f3d954f86bcdbfa91322a46c89f67", "main-result": "0f4aa84d0a01bba3302829e11a79635483cbc7ed332a7585666226d663e4536a", - "preamble": "b7cfbdccff4b2102e3394006c7bf8d47522397d055b3e8631e481a9db511d123", + "preamble": "65fac58b5272dd01e2452877dd1c25c81c390fb0eb3dc31389c0404b905cca31", "reproduce": "a3378bedac0f19d71a46de7ce86eb0d6c5bce141215fe1564c9ff7ad8b9e38a4", - "start-here": "661bcd24c5cef092cf4d995ca102f740ae62d0ac5b35c1383b421f101cc8ba50", - "status": "4247275cd63fe06bb87fd248db4543aca69f00da41fc15df46e492809dfca2d1" + "start-here": "58be1f12106bbb1d720d7268f1b316aa9e2658c3112b4b9cb50afcb54635467c", + "status": "824ad88aaf7e67026e0f6d90e306f3adac68b5168315286c6b828ec1ed9de6a2" } } }, "source": "README.md", - "source_sha256": "e7838cdb10b73590c18878a0cfde2c88b1a5c68fe2c68b122ea058f908032b4e", + "source_sha256": "c87be61823cff73f46f58c2248e4219e749d1358431804f63d2bdd3b0c76bb0b", "version": 1 } diff --git a/paper/manuscript_resource.md b/paper/manuscript_resource.md index dbc5d92..fab9eb5 100644 --- a/paper/manuscript_resource.md +++ b/paper/manuscript_resource.md @@ -10,11 +10,11 @@ Public availability of a neoantigen predictor does not ensure that its software, ### Methods -We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and an independent personalized-vaccine ELISPOT cohort; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. +We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and two endpoint-distinct personalized-vaccine ELISPOT cohorts; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. ### Results -All five pinned predictors produced outputs within their declared input support. The initial 520-row TESLA fixture was entirely training-overlapped and was retained only as a leakage-positive reproduction test. After excluding 45 exact PRIME2 overlaps from 17,520 IMPROVE records, 17,475 records from 70 patients remained. On common support, PRIME achieved AUROC 0.597 and mean pMHC-pair Recall@20 0.260 among 60 positive-bearing patients, versus 0.546 and 0.146 for BigMHC. Transparent peptide baselines outperformed HLA-only baselines under both patient- and study-held-out fitting, while adding HLA to peptide features did not consistently improve over peptide features alone. A frozen extension evaluated five models on 2,315 overlap-filtered vaccine peptides with a distinct post-vaccination ELISPOT endpoint. Support-matched random ranking showed that high marginal Top-K values did not necessarily imply useful ranking signal. +All five pinned predictors produced outputs within their declared input support. The initial 520-row TESLA fixture was entirely training-overlapped and was retained only as a leakage-positive reproduction test. After excluding 45 exact PRIME2 overlaps from 17,520 IMPROVE records, 17,475 records from 70 patients remained. On common support, PRIME achieved AUROC 0.597 and mean pMHC-pair Recall@20 0.260 among 60 positive-bearing patients, versus 0.546 and 0.146 for BigMHC. Transparent peptide baselines outperformed HLA-only baselines under both patient- and study-held-out fitting, while adding HLA to peptide features did not consistently improve over peptide features alone. A frozen extension evaluated five models on 2,315 overlap-filtered vaccine peptides with a distinct post-vaccination ELISPOT endpoint. A second endpoint-distinct vaccine cohort contributed 129 individually assayed short peptides from 9 patients. Support-matched and cross-domain analyses showed that high marginal Top-K values did not necessarily imply stable or useful ranking signal. ### Conclusions @@ -32,7 +32,7 @@ NeoRepro asks a narrower question than developing a new predictor: what evidence ### Study design and reproducibility contract -The core protocol was frozen in `RESEARCH_SPEC.md`; the external extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json` before extension inference. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. +The core protocol was frozen in `RESEARCH_SPEC.md`; the Zhao extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json`, and the RCC cohort in `research/extension_protocol_rcc_v1.json`, before their respective inference runs. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. ### Dataset construction and provenance @@ -42,6 +42,8 @@ For the independent extension, we extracted checksum-pinned `Table1.xlsx` from t The administered peptides had been selected by the source workflow using earlier NetMHC/NetMHCpan binding predictions and an IC50 threshold below 500 nM [@zhao2026vaccine]. Consequently, this cohort evaluates reranking within a predictor-enriched candidate set rather than unselected tumor mutations. Reported peptide–HLA restrictions were computational assignments used by the source analysis and were not individually established by HLA-blocking experiments. +For the endpoint-distinct RCC extension, we checksum-pinned Supplementary Table 2 from Braun and colleagues and retained 129 short-peptide rows from nine vaccinated patients after excluding one row without a usable short peptide/HLA assignment [@braun2025rcc]. Every retained row contained three peptide-stimulation and three matched no-stimulation replicates. Positivity followed the source individual-peptide p-value rule. The HLA field denotes the source-predicted best short-epitope allele; these rows therefore evaluate post-vaccination, in-vitro restimulation responses and cannot establish untreated intrinsic immunogenicity or natural tumour presentation. + ### Training-overlap policy The official PRIME2 supplementary archive and Table S4 workbook were downloaded by stable article identifier and checksum [@gfeller2023prime2]. We compared canonical peptides, peptide–HLA pairs and labels against every benchmark record and indexed same-HLA, same-length Hamming-distance-one training neighbors. Because the public BigMHC construction incorporates non-random PRIME1/2 peptide records of eligible length into immunogenicity training/validation, those exact matches were flagged separately [@albert2023bigmhc]. A single union exclusion of all exact PRIME2 peptide–HLA matches defined the common benchmark because peptide specificity is HLA-conditioned; peptides seen only with another HLA received a separate exclusion sensitivity, as did one-substitution neighbors. PRIME2 Table S4 lacks source mutation, patient and study identifiers, so those overlap dimensions are explicitly unavailable rather than assumed absent. The full benchmark and row-level audit remain versioned. @@ -56,7 +58,7 @@ To distinguish fixed pretrained tools from models with known splitting, we fitte ### Resource packaging, sensitivity and quality control -The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. +The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. A machine-validated extension contract defines Dataset Cards, Predictor Cards and prediction artifacts, while the expanded artifact census preserves successful, non-comparable and failed public-tool attempts. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Cross-dataset stability and first-place probabilities are explicitly exploratory and task-stratified. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. ## Results @@ -119,12 +121,31 @@ The prospectively frozen primary metric was patient-macro NDCG@5 because the med | DeepImmuno-CNN | 1,015 | 0.526 | 0.158 | 0.755 (0.691–0.816) | 0.759 | -0.004 | | PRIME | 2,310 | 0.531 | 0.148 | 0.604 (0.546–0.660) | 0.581 | 0.023 | +### Endpoint-distinct RCC vaccine cohort + +The separately frozen RCC protocol retained 129 individually assayed short peptides from 9 vaccinated patients after excluding one source row with no usable short peptide/HLA assignment [@braun2025rcc]. The assay compared three peptide-stimulation replicates with three matched no-stimulation replicates; labels follow the source p-value threshold and therefore are assay-context outcomes rather than untreated biological negatives. No exact PRIME2, BigMHC-construction or DeepImmuno training overlap was identified among the 129 records, while DeepHLApan row-level training identity remains unknown. + +On near-complete support, PRIME had AUROC 0.580 and patient NDCG@5 0.691; BigMHC had 0.476 and 0.533, respectively. DeepImmuno-CNN supported only 51 records. With nine patients, all estimates are descriptive and do not establish a cross-domain interaction, universal ordering, natural tumour presentation or clinical efficacy. + +**Table 4. RCC personalized-vaccine cohort.** The endpoint is post-vaccination, individual-peptide IFN-γ ELISpot after in-vitro stimulation. Random NDCG@5 is calculated on each model's exact support. + +| Predictor | Predicted records | AUROC | AP | Patient NDCG@5 (95% CI) | Random NDCG@5 | Gain over random | +|---|---:|---:|---:|---:|---:|---:| +| BigMHC | 128 | 0.476 | 0.586 | 0.533 (0.418–0.640) | 0.587 | -0.054 | +| DeepHLApan | 128 | 0.505 | 0.653 | 0.614 (0.407–0.765) | 0.587 | 0.027 | +| DeepImmuno-CNN | 51 | 0.472 | 0.583 | 0.711 (0.591–0.839) | 0.695 | 0.016 | +| PRIME | 128 | 0.580 | 0.639 | 0.691 (0.611–0.789) | 0.587 | 0.104 | + +### Expanded reproducibility profile and extension contract + +The artifact census now records 12 pinned predictor entries. Beyond the five benchmarked tools, 7 entries are retained as profile-only, non-comparable, pending or unreproducible outcomes rather than being silently omitted. The public extension contract supplies machine-validated Dataset Cards, Predictor Cards and prediction-artifact schemas together with a common-support evaluator; these additions improve reuse but do not make heterogeneous prediction tasks scientifically interchangeable. + ### Reusable benchmark outputs NeoRepro's primary output is the versioned evidence chain rather than a winner label. Each benchmark record can be traced to a source row and overlap classification; each score records predictor version, task, direction and status; model comparisons use explicit common support; and all reported numerical outputs are generated from frozen result files. The resource also preserves the fully overlapped TESLA fixture and failed NCI eligibility gate as negative controls, so future evaluations can test whether leakage and invalid-label safeguards behave as intended. Metric validation agreed with an independent implementation to floating-point precision, and clean-checkout reproduction regenerated the frozen outputs. -Figures 1–5 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity and the vaccine-cohort extension, respectively (`results/figures/`). +Figures 1–6 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity, the Zhao vaccine-cohort extension and exploratory endpoint/domain stability, respectively (`results/figures/` and `results/analysis/stability/`). ## Discussion @@ -132,11 +153,11 @@ NeoRepro's central contribution is a reusable measurement contract. The first se The benchmark's biological results are deliberately secondary to that contract. On the filtered IMPROVE common set, PRIME showed better pooled and patient-level point estimates than BigMHC within the same broad peptide–HLA immunogenicity-score category. Their training labels and score contracts differ, so this is an observation about the pinned implementations and evaluation contract, not evidence of universal superiority. MHCflurry addresses presentation rather than T-cell recognition, was invoked without flanking context, and was tested only for association with the recognition endpoint; its results cannot validate presentation performance. All absolute average-precision values were low in a highly imbalanced screen, and patient-level pMHC retrieval varied widely. -The independent vaccine cohort illustrates why support and task context belong in the resource. BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of the cohort, and its high marginal NDCG@5 did not exceed the higher random expectation on that restricted support. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, but Zhao AUROC did not resolve their difference, IMPROVE NDCG@5 did not resolve the opposite direction, no formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. +The vaccine cohorts illustrate why support and task context belong in the resource. In Zhao, BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of Zhao and RCC, and high marginal NDCG@5 on restricted support cannot be read as a general advantage. RCC was smaller still, with only nine patients. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, whereas RCC and IMPROVE produced different point-estimate patterns. No formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. The transparent baselines refine interpretation of HLA effects. HLA-only models exceeded chance modestly under LOPO but weakened under LOSO, consistent with cohort- and allele-associated label structure. Peptide features supplied more stable signal; adding HLA did not consistently improve beyond them. For BigMHC, 32.5% of observed score variance lay between HLA groups, but this unadjusted quantity also mixes peptide composition, patient, cohort and preselection; it is not an isolated allele effect. HLA sensitivity therefore requires more than one shortcut diagnostic. -This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao peptides were likewise selected using earlier binding predictors. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. One positive-bearing patient in the source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five tools are a judicious executable subset, not an exhaustive census; they cover only MHC-I and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extension is biologically complementary rather than a pure replication: vaccination and peptide-pulsed dendritic-cell administration can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. +This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao and RCC peptides were likewise selected within vaccine-design workflows. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support the original IMPROVE LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. RCC adds only nine vaccinated patients and uses predicted HLA assignments. One positive-bearing patient in the IMPROVE source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five benchmarked tools are a judicious executable subset; the broader census includes heterogeneous tools that cannot be placed into the same comparison. All cover only MHC-I here and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extensions are biologically complementary rather than pure replications: vaccination and ex-vivo or in-vitro stimulation can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. The practical implication is that neoantigen benchmarking should release source-grounded record identifiers, predictor versions, complete missingness, training-overlap audits, support-matched references and patient-level retrieval alongside pooled metrics. NeoRepro provides these components as an executable resource that can be extended with new datasets or predictors without erasing negative results. Its value is not a new ranking algorithm or a universal leaderboard; it is a reproducible way to determine which comparison claims survive provenance, leakage, support and patient-level checks. diff --git a/paper/manuscript_template.md b/paper/manuscript_template.md index c987edc..7472010 100644 --- a/paper/manuscript_template.md +++ b/paper/manuscript_template.md @@ -10,7 +10,7 @@ Public availability of a neoantigen predictor does not ensure that its software, ### Methods -We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and an independent personalized-vaccine ELISPOT cohort; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. +We pinned five public predictors in isolated environments; harmonized a presentation-prefiltered patient-matched pMHC multimer screen and two endpoint-distinct personalized-vaccine ELISPOT cohorts; and versioned record provenance, known training-overlap classifications, standardized prediction adapters and failures. Comparisons used common prediction support, pooled discrimination, patient-level Top-K retrieval, patient bootstrap, support-matched random ranking and transparent held-out baselines. ### Results @@ -32,7 +32,7 @@ NeoRepro asks a narrower question than developing a new predictor: what evidence ### Study design and reproducibility contract -The core protocol was frozen in `RESEARCH_SPEC.md`; the external extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json` before extension inference. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. +The core protocol was frozen in `RESEARCH_SPEC.md`; the Zhao extension and primary NDCG@5 endpoint were separately frozen in `research/extension_protocol.json`, and the RCC cohort in `research/extension_protocol_rcc_v1.json`, before their respective inference runs. Each predictor was assigned a pinned source revision, isolated environment, legal-access record, standardized adapter and evidence directory. Failed attempts were retained. MHCflurry 2.2.1 was run as a presentation predictor; BigMHC v1.0, PRIME 2.0, DeepImmuno-CNN and DeepHLApan were run as immunogenicity predictors. Scores were oriented so that larger values indicate stronger predicted evidence. No missing prediction was imputed. ### Dataset construction and provenance @@ -42,6 +42,8 @@ For the independent extension, we extracted checksum-pinned `Table1.xlsx` from t The administered peptides had been selected by the source workflow using earlier NetMHC/NetMHCpan binding predictions and an IC50 threshold below 500 nM [@zhao2026vaccine]. Consequently, this cohort evaluates reranking within a predictor-enriched candidate set rather than unselected tumor mutations. Reported peptide–HLA restrictions were computational assignments used by the source analysis and were not individually established by HLA-blocking experiments. +For the endpoint-distinct RCC extension, we checksum-pinned Supplementary Table 2 from Braun and colleagues and retained 129 short-peptide rows from nine vaccinated patients after excluding one row without a usable short peptide/HLA assignment [@braun2025rcc]. Every retained row contained three peptide-stimulation and three matched no-stimulation replicates. Positivity followed the source individual-peptide p-value rule. The HLA field denotes the source-predicted best short-epitope allele; these rows therefore evaluate post-vaccination, in-vitro restimulation responses and cannot establish untreated intrinsic immunogenicity or natural tumour presentation. + ### Training-overlap policy The official PRIME2 supplementary archive and Table S4 workbook were downloaded by stable article identifier and checksum [@gfeller2023prime2]. We compared canonical peptides, peptide–HLA pairs and labels against every benchmark record and indexed same-HLA, same-length Hamming-distance-one training neighbors. Because the public BigMHC construction incorporates non-random PRIME1/2 peptide records of eligible length into immunogenicity training/validation, those exact matches were flagged separately [@albert2023bigmhc]. A single union exclusion of all exact PRIME2 peptide–HLA matches defined the common benchmark because peptide specificity is HLA-conditioned; peptides seen only with another HLA received a separate exclusion sensitivity, as did one-substitution neighbors. PRIME2 Table S4 lacks source mutation, patient and study identifiers, so those overlap dimensions are explicitly unavailable rather than assumed absent. The full benchmark and row-level audit remain versioned. @@ -56,7 +58,7 @@ To distinguish fixed pretrained tools from models with known splitting, we fitte ### Resource packaging, sensitivity and quality control -The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. +The resource stores canonical schemas, source-row provenance, predictor and dataset registries, standardized prediction files, overlap classifications, model-support tables, paired comparisons, figures and a SHA-256 result manifest. A machine-validated extension contract defines Dataset Cards, Predictor Cards and prediction artifacts, while the expanded artifact census preserves successful, non-comparable and failed public-tool attempts. Third-party artifacts that cannot be redistributed are obtained from their official locations under their original terms. We reported per-study and per-HLA metrics, performance after replacing scores by within-HLA ranks, performance of HLA mean scores alone, and the score-scale-specific, unadjusted fraction of observed score variance lying between HLA groups. Cross-dataset stability and first-place probabilities are explicitly exploratory and task-stratified. Per-HLA tables include record, positive, patient and study support, and mark rows with fewer than three positives or three patients as unsupported for interpretation. HLA analyses are exploratory; no permutation p-values are used for biological inference. Deterministic unit tests cover schema counts, hashes, score direction, missingness, tie handling and metric fixtures. Pooled AUROC and average precision were cross-checked against scikit-learn, and the documented workflow was rerun from a clean checkout. ## Results @@ -66,7 +68,7 @@ The resource stores canonical schemas, source-row provenance, predictor and data NeoRepro's primary output is the versioned evidence chain rather than a winner label. Each benchmark record can be traced to a source row and overlap classification; each score records predictor version, task, direction and status; model comparisons use explicit common support; and all reported numerical outputs are generated from frozen result files. The resource also preserves the fully overlapped TESLA fixture and failed NCI eligibility gate as negative controls, so future evaluations can test whether leakage and invalid-label safeguards behave as intended. Metric validation agreed with an independent implementation to floating-point precision, and clean-checkout reproduction regenerated the frozen outputs. -Figures 1–5 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity and the vaccine-cohort extension, respectively (`results/figures/`). +Figures 1–6 are generated from frozen result files: fixed predictor performance, patient Recall@20, held-out baselines, HLA sensitivity, the Zhao vaccine-cohort extension and exploratory endpoint/domain stability, respectively (`results/figures/` and `results/analysis/stability/`). ## Discussion @@ -74,11 +76,11 @@ NeoRepro's central contribution is a reusable measurement contract. The first se The benchmark's biological results are deliberately secondary to that contract. On the filtered IMPROVE common set, PRIME showed better pooled and patient-level point estimates than BigMHC within the same broad peptide–HLA immunogenicity-score category. Their training labels and score contracts differ, so this is an observation about the pinned implementations and evaluation contract, not evidence of universal superiority. MHCflurry addresses presentation rather than T-cell recognition, was invoked without flanking context, and was tested only for association with the recognition endpoint; its results cannot validate presentation performance. All absolute average-precision values were low in a highly imbalanced screen, and patient-level pMHC retrieval varied widely. -The independent vaccine cohort illustrates why support and task context belong in the resource. BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of the cohort, and its high marginal NDCG@5 did not exceed the higher random expectation on that restricted support. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, but Zhao AUROC did not resolve their difference, IMPROVE NDCG@5 did not resolve the opposite direction, no formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. +The vaccine cohorts illustrate why support and task context belong in the resource. In Zhao, BigMHC had the largest positive NDCG@5 gain over random ranking among the near-complete-support immunogenicity models, while DeepHLApan was approximately at its random reference. DeepImmuno-CNN covered less than half of Zhao and RCC, and high marginal NDCG@5 on restricted support cannot be read as a general advantage. RCC was smaller still, with only nine patients. A paired BigMHC–PRIME contrast favored BigMHC for Zhao NDCG@5, whereas RCC and IMPROVE produced different point-estimate patterns. No formal dataset-by-model interaction was tested, and the reported pairwise intervals were not adjusted for multiple comparisons. We therefore do not claim a statistically established cross-domain ranking reversal. The defensible observation is narrower: model conclusions depend on endpoint, metric and supported candidate set, and no evaluated score was a stable universal winner. The transparent baselines refine interpretation of HLA effects. HLA-only models exceeded chance modestly under LOPO but weakened under LOSO, consistent with cohort- and allele-associated label structure. Peptide features supplied more stable signal; adding HLA did not consistently improve beyond them. For BigMHC, 32.5% of observed score variance lay between HLA groups, but this unadjusted quantity also mixes peptide composition, patient, cohort and preselection; it is not an isolated allele effect. HLA sensitivity therefore requires more than one shortcut diagnostic. -This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao peptides were likewise selected using earlier binding predictors. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. One positive-bearing patient in the source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five tools are a judicious executable subset, not an exhaustive census; they cover only MHC-I and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extension is biologically complementary rather than a pure replication: vaccination and peptide-pulsed dendritic-cell administration can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. +This study has important limitations. IMPROVE candidates were preselected through a presentation-oriented pipeline and are not a random sample of tumor mutations; the Zhao and RCC peptides were likewise selected within vaccine-design workflows. Our analysis therefore tests reranking after candidate-selection gates, not end-to-end discovery from all tumor variants. Experimental nonresponse is assay-, sample- and context-dependent, not proof that a peptide can never be immunogenic. PBMC, TIL and, in one cohort, TIL-ACT infusion-product sampling are not separated in the released canonical inputs used here. Identical peptide–HLA pairs had conflicting outcomes across patients, directly showing that recognition is not a deterministic function of the model inputs; treatment, tumor microenvironment and TCR repertoire were not modeled. Only three cohorts support the original IMPROVE LOSO analysis, and cohort simultaneously changes cancer, treatment, sample source and candidate-generation context. RCC adds only nine vaccinated patients and uses predicted HLA assignments. One positive-bearing patient in the IMPROVE source data lost all positives after common overlap exclusion, leaving 60 patients for ranking. Exact matching cannot detect undocumented training data or representation overlap. The five benchmarked tools are a judicious executable subset; the broader census includes heterogeneous tools that cannot be placed into the same comparison. All cover only MHC-I here and require different inputs and licenses. BigMHC and PRIME use only mutant peptide and HLA here, without wild-type counterpart, expression, clonality or direct presentation evidence, so they do not measure complete neoantigen quality. The vaccine extensions are biologically complementary rather than pure replications: vaccination and ex-vivo or in-vitro stimulation can induce or amplify responses absent in untreated disease, reported HLA restrictions were not individually proven experimentally, and DeepHLApan's public repository lacks a row-level training manifest. The clean reproduction is demonstrated on the documented platform, while upstream availability and platform compatibility can change. Finally, no computational benchmark here establishes vaccine efficacy, treatment response or clinical benefit. The practical implication is that neoantigen benchmarking should release source-grounded record identifiers, predictor versions, complete missingness, training-overlap audits, support-matched references and patient-level retrieval alongside pooled metrics. NeoRepro provides these components as an executable resource that can be extended with new datasets or predictors without erasing negative results. Its value is not a new ranking algorithm or a universal leaderboard; it is a reproducible way to determine which comparison claims survive provenance, leakage, support and patient-level checks. diff --git a/paper/references.bib b/paper/references.bib index f078800..5b2edb1 100644 --- a/paper/references.bib +++ b/paper/references.bib @@ -134,3 +134,13 @@ @article{paul2020benchmark pmid = {32453790}, pmcid = {PMC7274474} } +@article{braun2025rcc, + author = {Braun, David A. and others}, + title = {A neoantigen vaccine generates antitumour immunity in renal cell carcinoma}, + journal = {Nature}, + year = {2025}, + volume = {639}, + pages = {474--482}, + doi = {10.1038/s41586-024-08507-5}, + url = {https://doi.org/10.1038/s41586-024-08507-5} +} diff --git a/predictors/example/adapter.py b/predictors/example/adapter.py new file mode 100644 index 0000000..176ee8f --- /dev/null +++ b/predictors/example/adapter.py @@ -0,0 +1,17 @@ +"""Minimal third-party adapter template. Replace only `predict` and keep the artifact contract.""" +import csv, sys + +def predict(peptide: str, hla: str) -> float: + # TODO: call the pinned upstream predictor; never return a fabricated score. + raise NotImplementedError + +def main(input_csv, output_csv): + with open(input_csv, newline="", encoding="utf-8") as src, open(output_csv, "w", newline="", encoding="utf-8") as dst: + rows=list(csv.DictReader(src)); fields=["record_id","predictor","predictor_version","task","score","score_direction","status"] + out=csv.DictWriter(dst, fieldnames=fields); out.writeheader() + for row in rows: + try: score=predict(row["peptide"], row["hla"]); status="predicted" + except Exception: score=""; status="failed" + out.writerow({"record_id":row["record_id"],"predictor":"YOUR_ID","predictor_version":"YOUR_VERSION","task":"YOUR_TASK","score":score,"score_direction":"higher","status":status}) + +if __name__ == "__main__": main(sys.argv[1], sys.argv[2]) diff --git a/predictors/mhcmatch/attempts/20260821T065057715285Z-test.json b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.json new file mode 100644 index 0000000..288c06e --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.json @@ -0,0 +1,16 @@ +{ + "command": [ + "predictors/mhcmatch/.venv/bin/mhcmatch", + "--help" + ], + "duration_seconds": 1.92031, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "mhcmatch", + "python": "3.11.15", + "returncode": 0, + "stage": "test", + "started_at": "2026-08-21T06:50:57.715285+00:00", + "stderr_log": "20260821T065057715285Z-test.stderr.log", + "stdout_log": "20260821T065057715285Z-test.stdout.log", + "timed_out": false +} diff --git a/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stderr.log b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stdout.log b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stdout.log new file mode 100644 index 0000000..9344186 --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065057715285Z-test.stdout.log @@ -0,0 +1,49 @@ +usage: mhcmatch [-h] + {decompose,restriction,affinity,binder,scan,source,logo,span,predict,bootstrap,rank,explain,complement,neoag,mimicry,mimics,vector,deslip,expression} + ... + +peptide-MHC presentation tools + +positional arguments: + {decompose,restriction,affinity,binder,scan,source,logo,span,predict,bootstrap,rank,explain,complement,neoag,mimicry,mimics,vector,deslip,expression} + decompose split a peptide into anchor / TCR-facing parts (X + masks) + restriction rank presenting alleles for a peptide + affinity predict IC50 (nM) + neoantigen amplitude/DAI for a + peptide + binder generalized binder score (presentation x affinity) + ranked over alleles + scan find presented peptides in a protein (sequence or + FASTA path) + source find the self peptide a neoantigen derives from + logo motif logo (information content) + length distribution + span extend an MHC-II binding core to the full presented + ligand + predict score a variant peptide-window FASTA -> native + + .scored.csv + bootstrap pre-fetch the pmhc panel (and optionally proteomes) + from HF + rank rank neoantigen candidates (FASTA of windows, or a + scored table) + explain every component of the aggregate for one (peptide, + allele) + complement complementarity score (recognition axis) for peptides + — vectorised + neoag annotate candidates against the tested-neoantigen + database (nearest validated-immunogenic peptide + + substitution distance) + mimicry the fitted mimicry aggregate: signed viral / self / + thymus contributions per anchor and TCR-facing + channel, and their sum + mimics near-identical reference peptides per category (self / + thymus / viral / bacterial / neoag) -- batched and + threaded + vector assemble a polyepitope cassette: withdraw on safety, + choose how many units per allotype, order them, pick a + spacer + deslip find (and repair) the m1-pseudouridine +1 frameshift + motif in a cassette coding sequence + expression reference expression by normal tissue or tumour type + +options: + -h, --help show this help message and exit diff --git a/predictors/mhcmatch/attempts/20260821T065145972187Z-run.json b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.json new file mode 100644 index 0000000..f1e3ea8 --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.json @@ -0,0 +1,17 @@ +{ + "command": [ + "predictors/mhcmatch/.venv/bin/mhcmatch", + "binder", + "NLVPMVATV" + ], + "duration_seconds": 180.143593, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "mhcmatch", + "python": "3.11.15", + "returncode": 124, + "stage": "run", + "started_at": "2026-08-21T06:51:45.972187+00:00", + "stderr_log": "20260821T065145972187Z-run.stderr.log", + "stdout_log": "20260821T065145972187Z-run.stdout.log", + "timed_out": true +} diff --git a/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stderr.log b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stderr.log new file mode 100644 index 0000000..31ae88f --- /dev/null +++ b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stderr.log @@ -0,0 +1,3 @@ +Warning: You are sending unauthenticated requests to the HF Hub. Please set a HF_TOKEN to enable higher rate limits and faster downloads. + +Timed out after 180 seconds. \ No newline at end of file diff --git a/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stdout.log b/predictors/mhcmatch/attempts/20260821T065145972187Z-run.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/clone.stderr.log b/predictors/mhcmatch/attempts/clone.stderr.log new file mode 100644 index 0000000..5743f41 --- /dev/null +++ b/predictors/mhcmatch/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcmatch/source'... diff --git a/predictors/mhcmatch/attempts/clone.stdout.log b/predictors/mhcmatch/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/install.stderr.log b/predictors/mhcmatch/attempts/install.stderr.log new file mode 100644 index 0000000..0fcd9cb --- /dev/null +++ b/predictors/mhcmatch/attempts/install.stderr.log @@ -0,0 +1,26 @@ +Using Python 3.11.15 environment at: predictors/mhcmatch/.venv +Resolved 18 packages in 290ms + Building mhcmatch @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcmatch/source +Downloading seqtree (1.5MiB) + Downloaded seqtree + Built mhcmatch @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcmatch/source +Prepared 3 packages in 366ms +Installed 18 packages in 70ms + + anyio==4.14.2 + + certifi==2026.7.22 + + click==8.4.2 + + filelock==3.32.3 + + fsspec==2026.7.0 + + h11==0.16.0 + + hf-xet==1.6.0 + + httpcore==1.0.9 + + httpx==0.28.1 + + huggingface-hub==1.28.0 + + idna==3.19 + + mhcmatch==0.25.0 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcmatch/source) + + numpy==2.4.6 + + packaging==26.3 + + pyyaml==6.0.3 + + seqtree==0.7.0 + + tqdm==4.70.0 + + typing-extensions==4.16.0 diff --git a/predictors/mhcmatch/attempts/install.stdout.log b/predictors/mhcmatch/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcmatch/attempts/smoke.stderr.log b/predictors/mhcmatch/attempts/smoke.stderr.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/mhcmatch/attempts/smoke.stderr.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/mhcmatch/attempts/smoke.stdout.log b/predictors/mhcmatch/attempts/smoke.stdout.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/mhcmatch/attempts/smoke.stdout.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/mhcmatch/attempts/sweep_receipt.json b/predictors/mhcmatch/attempts/sweep_receipt.json new file mode 100644 index 0000000..21e458f --- /dev/null +++ b/predictors/mhcmatch/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "mhcmatch", + "revision": "956bdb613fdb1e128cbd7d59fabb9b47ef1aaae9", + "repo": "https://github.com/antigenomics/mhcmatch.git", + "install_returncode": 0, + "smoke_returncode": 125, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/mhcnuggets/adapter.py b/predictors/mhcnuggets/adapter.py new file mode 100644 index 0000000..5b699fe --- /dev/null +++ b/predictors/mhcnuggets/adapter.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Strict peptide-HLA adapter for the pinned MHCnuggets BA models. + +MHCnuggets emits IC50 in nM; lower is better. Unsupported alleles are not +silently rescued to the closest allele. +""" +from __future__ import annotations +import argparse, csv, re, subprocess, tempfile +from pathlib import Path + +VERSION = "2.4.0@b666fea3" +FIELDS = ["record_id", "predictor", "predictor_version", "task", "score", "score_direction", "status", "raw_score", "provenance_path"] + +def allele(value: str) -> str: + compact = re.sub(r"[^A-Z0-9]", "", value.upper().replace("HLA-", "").replace("HLA", "")) + m = re.fullmatch(r"([ABC])([0-9]{2})([0-9]{2})", compact) + return f"HLA-{m.group(1)}{m.group(2)}:{m.group(3)}" if m else "" + +def main() -> int: + ap = argparse.ArgumentParser(); ap.add_argument("--input", type=Path, required=True); ap.add_argument("--source-dir", type=Path, default=Path("predictors/mhcnuggets/source")); ap.add_argument("--output", type=Path, required=True); args = ap.parse_args() + source = args.source_dir.resolve(); observed = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=source, text=True).strip() + if observed != "b666fea3a54a1d357efba4ea4d8550ce5dd50aba": raise RuntimeError(f"unexpected revision {observed}") + rows = list(csv.DictReader(args.input.open(newline="", encoding="utf-8"))); required = {"record_id", "peptide", "hla"} + if not rows or required - set(rows[0]): raise ValueError(f"missing columns: {sorted(required-set(rows[0]))}") + out = {r["record_id"]: {"record_id": r["record_id"], "predictor": "MHCnuggets", "predictor_version": VERSION, "task": "binding", "score": "", "score_direction": "lower", "status": "unsupported_hla", "raw_score": "", "provenance_path": str(source)} for r in rows} + groups = {} + for r in rows: + h = allele(r["hla"]); model = source / "mhcnuggets" / "saves" / "production" / f"{h}_BA.h5" + if h and model.exists(): groups.setdefault(h, []).append(r) + with tempfile.TemporaryDirectory() as td: + for h, group in groups.items(): + pep = Path(td) / f"{h.replace(':','_')}.peps"; pred = Path(td) / f"{h.replace(':','_')}.csv"; pep.write_text("\n".join(r["peptide"] for r in group) + "\n", encoding="utf-8") + from mhcnuggets.src.predict import predict + predict(class_="I", peptides_path=str(pep), mhc=h, ba_models=True, output=str(pred)) + with pred.open(newline="", encoding="utf-8") as handle: + for r, p in zip(group, csv.DictReader(handle), strict=True): out[r["record_id"]].update(score=p["ic50"], raw_score=p["ic50"], status="predicted") + args.output.parent.mkdir(parents=True, exist_ok=True) + with args.output.open("w", newline="", encoding="utf-8") as handle: w = csv.DictWriter(handle, fieldnames=FIELDS, lineterminator="\n"); w.writeheader(); w.writerows(out.values()) + print(f"MHCnuggets {VERSION}: {sum(r['status']=='predicted' for r in out.values())}/{len(out)} predicted -> {args.output}"); return 0 +if __name__ == "__main__": raise SystemExit(main()) diff --git a/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.json b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.json new file mode 100644 index 0000000..d3ba42c --- /dev/null +++ b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.json @@ -0,0 +1,22 @@ +{ + "command": [ + "predictors/mhcnuggets/.venv/bin/python", + "predictors/mhcnuggets/adapter.py", + "--input", + "predictors/mhcnuggets/fixture.csv", + "--source-dir", + "predictors/mhcnuggets/source", + "--output", + "predictors/mhcnuggets/fixture_predictions.csv" + ], + "duration_seconds": 7.534415, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "mhcnuggets", + "python": "3.11.15", + "returncode": 0, + "stage": "adapter", + "started_at": "2026-08-21T06:50:01.940493+00:00", + "stderr_log": "20260821T065001940493Z-adapter.stderr.log", + "stdout_log": "20260821T065001940493Z-adapter.stdout.log", + "timed_out": false +} diff --git a/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stderr.log b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stderr.log new file mode 100644 index 0000000..e0aed8b --- /dev/null +++ b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stderr.log @@ -0,0 +1,2 @@ +/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/.venv/lib/python3.11/site-packages/keras/src/layers/core/masking.py:48: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead. + super().__init__(**kwargs) diff --git a/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stdout.log b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stdout.log new file mode 100644 index 0000000..b190a12 --- /dev/null +++ b/predictors/mhcnuggets/attempts/20260821T065001940493Z-adapter.stdout.log @@ -0,0 +1,7 @@ +Predicting for 3 peptides +Number of peptides skipped/total due to length 0 / 0 +Building model +Closest allele found HLA-A02:01 +Predicting with only binding affinity trained models +Writing output files... +MHCnuggets 2.4.0@b666fea3: 3/3 predicted -> predictors/mhcnuggets/fixture_predictions.csv diff --git a/predictors/mhcnuggets/attempts/clone.stderr.log b/predictors/mhcnuggets/attempts/clone.stderr.log new file mode 100644 index 0000000..3f1c808 --- /dev/null +++ b/predictors/mhcnuggets/attempts/clone.stderr.log @@ -0,0 +1,105 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source'... +Updating files: 0% (2/354) +Updating files: 1% (4/354) +Updating files: 2% (8/354) +Updating files: 3% (11/354) +Updating files: 4% (15/354) +Updating files: 5% (18/354) +Updating files: 5% (20/354) +Updating files: 6% (22/354) +Updating files: 7% (25/354) +Updating files: 8% (29/354) +Updating files: 9% (32/354) +Updating files: 10% (36/354) +Updating files: 11% (39/354) +Updating files: 12% (43/354) +Updating files: 13% (47/354) +Updating files: 14% (50/354) +Updating files: 15% (54/354) +Updating files: 16% (57/354) +Updating files: 17% (61/354) +Updating files: 18% (64/354) +Updating files: 19% (68/354) +Updating files: 20% (71/354) +Updating files: 21% (75/354) +Updating files: 22% (78/354) +Updating files: 23% (82/354) +Updating files: 24% (85/354) +Updating files: 25% (89/354) +Updating files: 26% (93/354) +Updating files: 27% (96/354) +Updating files: 28% (100/354) +Updating files: 29% (103/354) +Updating files: 30% (107/354) +Updating files: 31% (110/354) +Updating files: 32% (114/354) +Updating files: 33% (117/354) +Updating files: 34% (121/354) +Updating files: 35% (124/354) +Updating files: 36% (128/354) +Updating files: 37% (131/354) +Updating files: 38% (135/354) +Updating files: 39% (139/354) +Updating files: 40% (142/354) +Updating files: 41% (146/354) +Updating files: 42% (149/354) +Updating files: 43% (153/354) +Updating files: 44% (156/354) +Updating files: 45% (160/354) +Updating files: 46% (163/354) +Updating files: 47% (167/354) +Updating files: 48% (170/354) +Updating files: 48% (171/354) +Updating files: 49% (174/354) +Updating files: 50% (177/354) +Updating files: 51% (181/354) +Updating files: 52% (185/354) +Updating files: 53% (188/354) +Updating files: 54% (192/354) +Updating files: 55% (195/354) +Updating files: 56% (199/354) +Updating files: 57% (202/354) +Updating files: 58% (206/354) +Updating files: 59% (209/354) +Updating files: 60% (213/354) +Updating files: 61% (216/354) +Updating files: 62% (220/354) +Updating files: 63% (224/354) +Updating files: 64% (227/354) +Updating files: 65% (231/354) +Updating files: 66% (234/354) +Updating files: 67% (238/354) +Updating files: 68% (241/354) +Updating files: 69% (245/354) +Updating files: 70% (248/354) +Updating files: 71% (252/354) +Updating files: 72% (255/354) +Updating files: 73% (259/354) +Updating files: 74% (262/354) +Updating files: 75% (266/354) +Updating files: 76% (270/354) +Updating files: 77% (273/354) +Updating files: 78% (277/354) +Updating files: 79% (280/354) +Updating files: 80% (284/354) +Updating files: 81% (287/354) +Updating files: 82% (291/354) +Updating files: 83% (294/354) +Updating files: 84% (298/354) +Updating files: 85% (301/354) +Updating files: 86% (305/354) +Updating files: 87% (308/354) +Updating files: 88% (312/354) +Updating files: 89% (316/354) +Updating files: 90% (319/354) +Updating files: 91% (323/354) +Updating files: 92% (326/354) +Updating files: 93% (330/354) +Updating files: 94% (333/354) +Updating files: 95% (337/354) +Updating files: 96% (340/354) +Updating files: 97% (344/354) +Updating files: 98% (347/354) +Updating files: 99% (351/354) +Updating files: 100% (354/354) +Updating files: 100% (354/354), done. diff --git a/predictors/mhcnuggets/attempts/clone.stdout.log b/predictors/mhcnuggets/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcnuggets/attempts/install.stderr.log b/predictors/mhcnuggets/attempts/install.stderr.log new file mode 100644 index 0000000..6aa1123 --- /dev/null +++ b/predictors/mhcnuggets/attempts/install.stderr.log @@ -0,0 +1,8 @@ +Using Python 3.11.15 environment at: predictors/mhcnuggets/.venv +Resolved 56 packages in 590ms + Building mhcnuggets @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcnuggets/source + Built mhcnuggets @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcnuggets/source +Prepared 1 package in 246ms +Uninstalled 1 package in 2ms +Installed 1 package in 2ms + ~ mhcnuggets==2.4.0 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/mhcnuggets/source) diff --git a/predictors/mhcnuggets/attempts/install.stdout.log b/predictors/mhcnuggets/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcnuggets/attempts/smoke.stderr.log b/predictors/mhcnuggets/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/mhcnuggets/attempts/smoke.stdout.log b/predictors/mhcnuggets/attempts/smoke.stdout.log new file mode 100644 index 0000000..40a891b --- /dev/null +++ b/predictors/mhcnuggets/attempts/smoke.stdout.log @@ -0,0 +1 @@ +import ok diff --git a/predictors/mhcnuggets/attempts/sweep_receipt.json b/predictors/mhcnuggets/attempts/sweep_receipt.json new file mode 100644 index 0000000..a90f7d6 --- /dev/null +++ b/predictors/mhcnuggets/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "mhcnuggets", + "revision": "b666fea3a54a1d357efba4ea4d8550ce5dd50aba", + "repo": "https://github.com/KarchinLab/mhcnuggets.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/mhcnuggets/fixture.csv b/predictors/mhcnuggets/fixture.csv new file mode 100644 index 0000000..f5b7d6b --- /dev/null +++ b/predictors/mhcnuggets/fixture.csv @@ -0,0 +1,4 @@ +record_id,peptide,hla +mhcnuggets-fixture-1,AIAACAMLLV,HLA-A*02:01 +mhcnuggets-fixture-2,ALVCYIVMPV,HLA-A*02:01 +mhcnuggets-fixture-3,ALEPRKEIDV,HLA-A*02:01 diff --git a/predictors/mhcnuggets/fixture_predictions.csv b/predictors/mhcnuggets/fixture_predictions.csv new file mode 100644 index 0000000..9e173ec --- /dev/null +++ b/predictors/mhcnuggets/fixture_predictions.csv @@ -0,0 +1,4 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,raw_score,provenance_path +mhcnuggets-fixture-1,MHCnuggets,2.4.0@b666fea3,binding,49.76,lower,predicted,49.76,/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source +mhcnuggets-fixture-2,MHCnuggets,2.4.0@b666fea3,binding,13.9,lower,predicted,13.9,/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source +mhcnuggets-fixture-3,MHCnuggets,2.4.0@b666fea3,binding,5707.79,lower,predicted,5707.79,/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/mhcnuggets/source diff --git a/predictors/neofox/attempts/20260821T065057715299Z-test.json b/predictors/neofox/attempts/20260821T065057715299Z-test.json new file mode 100644 index 0000000..0cd2d46 --- /dev/null +++ b/predictors/neofox/attempts/20260821T065057715299Z-test.json @@ -0,0 +1,16 @@ +{ + "command": [ + "predictors/neofox/.venv/bin/neofox", + "--help" + ], + "duration_seconds": 41.265039, + "platform": "macOS-15.3-arm64-arm-64bit", + "predictor": "neofox", + "python": "3.11.15", + "returncode": 0, + "stage": "test", + "started_at": "2026-08-21T06:50:57.715299+00:00", + "stderr_log": "20260821T065057715299Z-test.stderr.log", + "stdout_log": "20260821T065057715299Z-test.stdout.log", + "timed_out": false +} diff --git a/predictors/neofox/attempts/20260821T065057715299Z-test.stderr.log b/predictors/neofox/attempts/20260821T065057715299Z-test.stderr.log new file mode 100644 index 0000000..fa570c8 --- /dev/null +++ b/predictors/neofox/attempts/20260821T065057715299Z-test.stderr.log @@ -0,0 +1,4 @@ +/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neofox/source/neofox/model/conversion.py:203: SyntaxWarning: assertion is always true, perhaps remove parentheses? + assert(mhc in [MHC_I, MHC_II], 'Bad MHC value') +/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neofox/source/neofox/model/conversion.py:260: SyntaxWarning: assertion is always true, perhaps remove parentheses? + assert (mhc in [MHC_I, MHC_II], 'Bad MHC value') diff --git a/predictors/neofox/attempts/20260821T065057715299Z-test.stdout.log b/predictors/neofox/attempts/20260821T065057715299Z-test.stdout.log new file mode 100644 index 0000000..49ea27c --- /dev/null +++ b/predictors/neofox/attempts/20260821T065057715299Z-test.stdout.log @@ -0,0 +1,47 @@ +usage: neofox [-h] --input-file INPUT_FILE --patient-data PATIENTS_DATA + --output-folder OUTPUT_FOLDER [--output-prefix OUTPUT_PREFIX] + [--with-all-neoepitopes] + [--rank-mhci-threshold RANK_MHCI_THRESHOLD] + [--rank-mhcii-threshold RANK_MHCII_THRESHOLD] + [--num-cpus NUM_CPUS] [--config CONFIG] + [--organism {human,mouse}] [--verbose] + +NeoFox 1.2.3 annotates a given set of neoantigen candidate sequences derived +from point mutation with relevant neoantigen features + +options: + -h, --help show this help message and exit + --input-file INPUT_FILE + Input file with neoantigens candidates represented by + long mutated peptide sequences. Supported formats: + tab-separated columns (extensions: .txt or .tsv) or + JSON (extension: .json) + --patient-data PATIENTS_DATA + file with data for patients with columns: identifier, + estimated_tumor_content, mhc_i_alleles, + mhc_ii_alleles, tissue + --output-folder OUTPUT_FOLDER + output folder + --output-prefix OUTPUT_PREFIX + prefix to name output files in the output folder + --with-all-neoepitopes + output annotations for all MHC-I and MHC-II + neoepitopes on all HLA alleles + --rank-mhci-threshold RANK_MHCI_THRESHOLD + MHC-I epitopes with a netMHCpan predicted rank greater + than or equal than this threshold will be filtered out + (default: 2.0) + --rank-mhcii-threshold RANK_MHCII_THRESHOLD + MHC-II epitopes with a netMHCIIpan predicted rank + greater than or equal than this threshold will be + filtered out (default: 5.0) + --num-cpus NUM_CPUS number of CPUs for computation + --config CONFIG an optional configuration file with all the + environment variables + --organism {human,mouse} + the organism to which the data corresponds + --verbose verbose logs + +NeoFox (NEOantigen Feature toolbOX) 1.2.3. Copyright (c) 2020-2021 TRON - +Translational Oncology at the University Medical Center of the Johannes +Gutenberg University Mainz gGmbH, all rights reserved diff --git a/predictors/neofox/attempts/clone.stderr.log b/predictors/neofox/attempts/clone.stderr.log new file mode 100644 index 0000000..5e7eb7d --- /dev/null +++ b/predictors/neofox/attempts/clone.stderr.log @@ -0,0 +1,103 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neofox/source'... +Updating files: 0% (2/329) +Updating files: 1% (4/329) +Updating files: 2% (7/329) +Updating files: 3% (10/329) +Updating files: 4% (14/329) +Updating files: 5% (17/329) +Updating files: 6% (20/329) +Updating files: 7% (24/329) +Updating files: 8% (27/329) +Updating files: 9% (30/329) +Updating files: 10% (33/329) +Updating files: 11% (37/329) +Updating files: 12% (40/329) +Updating files: 13% (43/329) +Updating files: 14% (47/329) +Updating files: 15% (50/329) +Updating files: 16% (53/329) +Updating files: 17% (56/329) +Updating files: 18% (60/329) +Updating files: 19% (63/329) +Updating files: 20% (66/329) +Updating files: 21% (70/329) +Updating files: 22% (73/329) +Updating files: 23% (76/329) +Updating files: 24% (79/329) +Updating files: 25% (83/329) +Updating files: 26% (86/329) +Updating files: 27% (89/329) +Updating files: 28% (93/329) +Updating files: 29% (96/329) +Updating files: 30% (99/329) +Updating files: 31% (102/329) +Updating files: 32% (106/329) +Updating files: 33% (109/329) +Updating files: 34% (112/329) +Updating files: 35% (116/329) +Updating files: 36% (119/329) +Updating files: 37% (122/329) +Updating files: 38% (126/329) +Updating files: 39% (129/329) +Updating files: 40% (132/329) +Updating files: 41% (135/329) +Updating files: 42% (139/329) +Updating files: 43% (142/329) +Updating files: 44% (145/329) +Updating files: 45% (149/329) +Updating files: 46% (152/329) +Updating files: 47% (155/329) +Updating files: 48% (158/329) +Updating files: 49% (162/329) +Updating files: 50% (165/329) +Updating files: 51% (168/329) +Updating files: 52% (172/329) +Updating files: 53% (175/329) +Updating files: 54% (178/329) +Updating files: 55% (181/329) +Updating files: 56% (185/329) +Updating files: 57% (188/329) +Updating files: 58% (191/329) +Updating files: 59% (195/329) +Updating files: 60% (198/329) +Updating files: 61% (201/329) +Updating files: 62% (204/329) +Updating files: 63% (208/329) +Updating files: 64% (211/329) +Updating files: 65% (214/329) +Updating files: 66% (218/329) +Updating files: 67% (221/329) +Updating files: 68% (224/329) +Updating files: 69% (228/329) +Updating files: 70% (231/329) +Updating files: 71% (234/329) +Updating files: 72% (237/329) +Updating files: 73% (241/329) +Updating files: 74% (244/329) +Updating files: 75% (247/329) +Updating files: 76% (251/329) +Updating files: 77% (254/329) +Updating files: 78% (257/329) +Updating files: 79% (260/329) +Updating files: 80% (264/329) +Updating files: 81% (267/329) +Updating files: 82% (270/329) +Updating files: 83% (274/329) +Updating files: 84% (277/329) +Updating files: 85% (280/329) +Updating files: 86% (283/329) +Updating files: 87% (287/329) +Updating files: 88% (290/329) +Updating files: 89% (293/329) +Updating files: 90% (297/329) +Updating files: 91% (300/329) +Updating files: 92% (303/329) +Updating files: 93% (306/329) +Updating files: 94% (310/329) +Updating files: 95% (313/329) +Updating files: 96% (316/329) +Updating files: 97% (320/329) +Updating files: 98% (323/329) +Updating files: 99% (326/329) +Updating files: 100% (329/329) +Updating files: 100% (329/329), done. diff --git a/predictors/neofox/attempts/clone.stdout.log b/predictors/neofox/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neofox/attempts/install.stderr.log b/predictors/neofox/attempts/install.stderr.log new file mode 100644 index 0000000..ea9c9ce --- /dev/null +++ b/predictors/neofox/attempts/install.stderr.log @@ -0,0 +1,8 @@ +Using Python 3.11.15 environment at: predictors/neofox/.venv +Resolved 44 packages in 356ms + Building neofox @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/neofox/source + Built neofox @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/neofox/source +Prepared 1 package in 269ms +Uninstalled 1 package in 7ms +Installed 1 package in 18ms + ~ neofox==1.2.3 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/neofox/source) diff --git a/predictors/neofox/attempts/install.stdout.log b/predictors/neofox/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neofox/attempts/smoke.stderr.log b/predictors/neofox/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neofox/attempts/smoke.stdout.log b/predictors/neofox/attempts/smoke.stdout.log new file mode 100644 index 0000000..40a891b --- /dev/null +++ b/predictors/neofox/attempts/smoke.stdout.log @@ -0,0 +1 @@ +import ok diff --git a/predictors/neofox/attempts/sweep_receipt.json b/predictors/neofox/attempts/sweep_receipt.json new file mode 100644 index 0000000..9152fd9 --- /dev/null +++ b/predictors/neofox/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "neofox", + "revision": "dd894a67ad02a4377ed3e4b1e44bd576360c98cd", + "repo": "https://github.com/TRON-Bioinformatics/neofox.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/neoguider/attempts/clone.stderr.log b/predictors/neoguider/attempts/clone.stderr.log new file mode 100644 index 0000000..61986ef --- /dev/null +++ b/predictors/neoguider/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/neoguider/source'... diff --git a/predictors/neoguider/attempts/clone.stdout.log b/predictors/neoguider/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neoguider/attempts/install.stderr.log b/predictors/neoguider/attempts/install.stderr.log new file mode 100644 index 0000000..4c76f1c --- /dev/null +++ b/predictors/neoguider/attempts/install.stderr.log @@ -0,0 +1,5 @@ +Using Python 3.11.15 environment at: predictors/neoguider/.venv +error: /Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research + Project/predictors/neoguider/source does not appear to be a Python + project, as neither `pyproject.toml` nor `setup.py` are present in the + directory diff --git a/predictors/neoguider/attempts/install.stdout.log b/predictors/neoguider/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/neoguider/attempts/smoke.stderr.log b/predictors/neoguider/attempts/smoke.stderr.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/neoguider/attempts/smoke.stderr.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/neoguider/attempts/smoke.stdout.log b/predictors/neoguider/attempts/smoke.stdout.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/neoguider/attempts/smoke.stdout.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/neoguider/attempts/sweep_receipt.json b/predictors/neoguider/attempts/sweep_receipt.json new file mode 100644 index 0000000..4790412 --- /dev/null +++ b/predictors/neoguider/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "neoguider", + "revision": "f95a5713453910902d657f1d148568b93c81026f", + "repo": "https://github.com/XuegongLab/neoguider.git", + "install_returncode": 2, + "smoke_returncode": 125, + "license_files": [ + "LICENSE.txt" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/pvactools/attempts/clone.stderr.log b/predictors/pvactools/attempts/clone.stderr.log new file mode 100644 index 0000000..22e82b4 --- /dev/null +++ b/predictors/pvactools/attempts/clone.stderr.log @@ -0,0 +1,110 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/pvactools/source'... +Updating files: 0% (2/4089) +Updating files: 1% (41/4089) +Updating files: 2% (82/4089) +Updating files: 3% (123/4089) +Updating files: 4% (164/4089) +Updating files: 5% (205/4089) +Updating files: 6% (246/4089) +Updating files: 6% (247/4089) +Updating files: 7% (287/4089) +Updating files: 8% (328/4089) +Updating files: 9% (369/4089) +Updating files: 10% (409/4089) +Updating files: 11% (450/4089) +Updating files: 12% (491/4089) +Updating files: 13% (532/4089) +Updating files: 14% (573/4089) +Updating files: 15% (614/4089) +Updating files: 16% (655/4089) +Updating files: 17% (696/4089) +Updating files: 18% (737/4089) +Updating files: 19% (777/4089) +Updating files: 20% (818/4089) +Updating files: 21% (859/4089) +Updating files: 22% (900/4089) +Updating files: 23% (941/4089) +Updating files: 24% (982/4089) +Updating files: 25% (1023/4089) +Updating files: 26% (1064/4089) +Updating files: 26% (1093/4089) +Updating files: 27% (1105/4089) +Updating files: 28% (1145/4089) +Updating files: 29% (1186/4089) +Updating files: 30% (1227/4089) +Updating files: 31% (1268/4089) +Updating files: 31% (1274/4089) +Updating files: 32% (1309/4089) +Updating files: 33% (1350/4089) +Updating files: 34% (1391/4089) +Updating files: 35% (1432/4089) +Updating files: 36% (1473/4089) +Updating files: 37% (1513/4089) +Updating files: 38% (1554/4089) +Updating files: 39% (1595/4089) +Updating files: 40% (1636/4089) +Updating files: 41% (1677/4089) +Updating files: 42% (1718/4089) +Updating files: 43% (1759/4089) +Updating files: 44% (1800/4089) +Updating files: 45% (1841/4089) +Updating files: 46% (1881/4089) +Updating files: 47% (1922/4089) +Updating files: 48% (1963/4089) +Updating files: 49% (2004/4089) +Updating files: 50% (2045/4089) +Updating files: 51% (2086/4089) +Updating files: 52% (2127/4089) +Updating files: 53% (2168/4089) +Updating files: 54% (2209/4089) +Updating files: 55% (2249/4089) +Updating files: 56% (2290/4089) +Updating files: 57% (2331/4089) +Updating files: 57% (2352/4089) +Updating files: 58% (2372/4089) +Updating files: 59% (2413/4089) +Updating files: 60% (2454/4089) +Updating files: 61% (2495/4089) +Updating files: 62% (2536/4089) +Updating files: 63% (2577/4089) +Updating files: 64% (2617/4089) +Updating files: 65% (2658/4089) +Updating files: 66% (2699/4089) +Updating files: 67% (2740/4089) +Updating files: 68% (2781/4089) +Updating files: 69% (2822/4089) +Updating files: 70% (2863/4089) +Updating files: 71% (2904/4089) +Updating files: 72% (2945/4089) +Updating files: 73% (2985/4089) +Updating files: 74% (3026/4089) +Updating files: 75% (3067/4089) +Updating files: 76% (3108/4089) +Updating files: 77% (3149/4089) +Updating files: 78% (3190/4089) +Updating files: 79% (3231/4089) +Updating files: 80% (3272/4089) +Updating files: 81% (3313/4089) +Updating files: 82% (3353/4089) +Updating files: 83% (3394/4089) +Updating files: 84% (3435/4089) +Updating files: 85% (3476/4089) +Updating files: 86% (3517/4089) +Updating files: 87% (3558/4089) +Updating files: 88% (3599/4089) +Updating files: 89% (3640/4089) +Updating files: 90% (3681/4089) +Updating files: 91% (3721/4089) +Updating files: 91% (3748/4089) +Updating files: 92% (3762/4089) +Updating files: 93% (3803/4089) +Updating files: 93% (3808/4089) +Updating files: 94% (3844/4089) +Updating files: 95% (3885/4089) +Updating files: 96% (3926/4089) +Updating files: 97% (3967/4089) +Updating files: 97% (3975/4089) +Updating files: 98% (4008/4089) +Updating files: 99% (4049/4089) +Updating files: 100% (4089/4089) +Updating files: 100% (4089/4089), done. diff --git a/predictors/pvactools/attempts/clone.stdout.log b/predictors/pvactools/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/pvactools/attempts/install.stderr.log b/predictors/pvactools/attempts/install.stderr.log new file mode 100644 index 0000000..2174ec3 --- /dev/null +++ b/predictors/pvactools/attempts/install.stderr.log @@ -0,0 +1,8 @@ +Using Python 3.11.15 environment at: predictors/pvactools/.venv +Resolved 107 packages in 748ms + Building pvactools @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/pvactools/source + Built pvactools @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/pvactools/source +Prepared 1 package in 607ms +Uninstalled 1 package in 2ms +Installed 1 package in 5ms + ~ pvactools==7.1.2 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/pvactools/source) diff --git a/predictors/pvactools/attempts/install.stdout.log b/predictors/pvactools/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/pvactools/attempts/smoke.stderr.log b/predictors/pvactools/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/pvactools/attempts/smoke.stdout.log b/predictors/pvactools/attempts/smoke.stdout.log new file mode 100644 index 0000000..38a22ff --- /dev/null +++ b/predictors/pvactools/attempts/smoke.stdout.log @@ -0,0 +1,49 @@ +usage: pvacseq [-h] + {run,binding_filter,coverage_filter,transcript_filter,top_score_filter,aggregate_report_filter,net_chop,netmhc_stab,calculate_reference_proteome_similarity,generate_protein_fasta,create_peptide_ordering_form,generate_aggregated_report,identify_problematic_amino_acids,mark_genes_of_interest,add_ml_predictions,update_tiers,download_example_data,install_vep_plugin} + ... + +positional arguments: + {run,binding_filter,coverage_filter,transcript_filter,top_score_filter,aggregate_report_filter,net_chop,netmhc_stab,calculate_reference_proteome_similarity,generate_protein_fasta,create_peptide_ordering_form,generate_aggregated_report,identify_problematic_amino_acids,mark_genes_of_interest,add_ml_predictions,update_tiers,download_example_data,install_vep_plugin} + run Run the pVACseq pipeline. + binding_filter Filter variants processed by IEDB by binding score. + coverage_filter Filter variants processed by IEDB by coverage, vaf, + and gene expression. + transcript_filter Filter variant transcripts processed by IEDB. + top_score_filter Pick the best neoepitope for each variant. + aggregate_report_filter + Filter an aggregate report and its metrics.json file + based on the variant Tier. + net_chop Run NetChop on existing pVACseq output .tsv to predict + cleavage sites on the neoepitopes. + netmhc_stab Run NetMHCStabPan on existing pVACseq output .tsv to + add stability predictions to the neoepitopes. + calculate_reference_proteome_similarity + Blast peptides against the reference proteome on + existing pVACseq output .tsv. + generate_protein_fasta + Generate an annotated fasta file from a VCF with + protein sequences of mutations and matching wildtypes. + create_peptide_ordering_form + Generate a peptide ordering form with coloring. + generate_aggregated_report + Generate an aggregated report from a pVACseq + .all_epitopes.tsv report file. + identify_problematic_amino_acids + Mark problematic amino acid positions in each epitope + or filter entries that have problematic amino acids. + mark_genes_of_interest + Mark predictions resulting from variants on a genes of + interest list. + add_ml_predictions Add ML-based neoantigen evaluation predictions to + pVACtools output files. + update_tiers Update tiers in an aggregated report in order to, for + example, use different thresholds or account for + problematic position or reference match information if + run after initial pipeline run. + download_example_data + Download example input and output files. + install_vep_plugin Install the Wildtype and Frameshift VEP plugins into + your VEP_plugins directory. + +options: + -h, --help show this help message and exit diff --git a/predictors/pvactools/attempts/sweep_receipt.json b/predictors/pvactools/attempts/sweep_receipt.json new file mode 100644 index 0000000..6cadce2 --- /dev/null +++ b/predictors/pvactools/attempts/sweep_receipt.json @@ -0,0 +1,13 @@ +{ + "predictor": "pvactools", + "revision": "2ce871be5ae56af141fe723d60d3841125874a74", + "repo": "https://github.com/griffithlab/pVACtools.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE", + "docs/license.rst" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/seq2neo/attempts/clone.stderr.log b/predictors/seq2neo/attempts/clone.stderr.log new file mode 100644 index 0000000..9c0bd54 --- /dev/null +++ b/predictors/seq2neo/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/seq2neo/source'... diff --git a/predictors/seq2neo/attempts/clone.stdout.log b/predictors/seq2neo/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/seq2neo/attempts/install.stderr.log b/predictors/seq2neo/attempts/install.stderr.log new file mode 100644 index 0000000..90236d5 --- /dev/null +++ b/predictors/seq2neo/attempts/install.stderr.log @@ -0,0 +1,9 @@ +Using Python 3.11.15 environment at: predictors/seq2neo/.venv + × No solution found when resolving dependencies: + ╰─▶ Because tensorflow==2.3.0 has no wheels with a matching Python ABI tag + (e.g., `cp311`) and seq2neo==2.1 depends on tensorflow==2.3.0, we can + conclude that seq2neo==2.1 cannot be used. + And because only seq2neo==2.1 is available and you require seq2neo, we + can conclude that your requirements are unsatisfiable. + +hint: You require CPython 3.11 (`cp311`), but we only found wheels for `tensorflow` (v2.3.0) with the following Python ABI tags: `cp35m`, `cp36m`, `cp37m`, `cp38` diff --git a/predictors/seq2neo/attempts/install.stdout.log b/predictors/seq2neo/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/seq2neo/attempts/smoke.stderr.log b/predictors/seq2neo/attempts/smoke.stderr.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/seq2neo/attempts/smoke.stderr.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/seq2neo/attempts/smoke.stdout.log b/predictors/seq2neo/attempts/smoke.stdout.log new file mode 100644 index 0000000..f719df2 --- /dev/null +++ b/predictors/seq2neo/attempts/smoke.stdout.log @@ -0,0 +1 @@ +not attempted \ No newline at end of file diff --git a/predictors/seq2neo/attempts/sweep_receipt.json b/predictors/seq2neo/attempts/sweep_receipt.json new file mode 100644 index 0000000..7259312 --- /dev/null +++ b/predictors/seq2neo/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "seq2neo", + "revision": "f342aa4ffa9012fb5509914333b9401a1eb286c4", + "repo": "https://github.com/XSLiuLab/Seq2Neo.git", + "install_returncode": 1, + "smoke_returncode": 125, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/predictors/vaxrank/attempts/clone.stderr.log b/predictors/vaxrank/attempts/clone.stderr.log new file mode 100644 index 0000000..2835688 --- /dev/null +++ b/predictors/vaxrank/attempts/clone.stderr.log @@ -0,0 +1 @@ +Cloning into '/Users/steve/Documents/ChatGPT/NeoRepro — Fully Autonomous Research Project/predictors/vaxrank/source'... diff --git a/predictors/vaxrank/attempts/clone.stdout.log b/predictors/vaxrank/attempts/clone.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/vaxrank/attempts/install.stderr.log b/predictors/vaxrank/attempts/install.stderr.log new file mode 100644 index 0000000..3d64e42 --- /dev/null +++ b/predictors/vaxrank/attempts/install.stderr.log @@ -0,0 +1,106 @@ +Using Python 3.11.15 environment at: predictors/vaxrank/.venv +Resolved 90 packages in 1.65s + Building vaxrank @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/vaxrank/source + Building matplotlib-venn==1.1.2 + Building docopt==0.6.2 +Downloading astropy (6.3MiB) +Downloading pyphen (2.0MiB) +Downloading oncoref (2.8MiB) + Downloaded pyphen + Built docopt==0.6.2 + Downloaded oncoref + Built vaxrank @ file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/vaxrank/source + Built matplotlib-venn==1.1.2 + Downloaded astropy +Prepared 25 packages in 552ms +Installed 90 packages in 917ms + + adjusttext==1.4.0 + + appdirs==1.4.4 + + argcomplete==3.7.2 + + astropy==8.0.1 + + astropy-iers-data==0.2026.8.18.14.22.31 + + biopython==1.88 + + brotli==1.2.0 + + certifi==2026.7.22 + + cffi==2.1.1 + + charset-normalizer==3.5.1 + + contourpy==1.3.3 + + cssselect2==0.9.0 + + cycler==0.12.1 + + datacache==1.7.0 + + dnachisel==3.2.16 + + docopt==0.6.2 + + et-xmlfile==2.0.0 + + filelock==3.32.3 + + flametree==0.2.1 + + fonttools==4.63.0 + + fsspec==2026.7.0 + + gtfparse==2.8.0 + + idna==3.19 + + isovar==1.7.2 + + jinja2==3.1.6 + + joblib==1.5.3 + + kiwisolver==1.5.0 + + markupsafe==3.0.3 + + matplotlib==3.11.1 + + matplotlib-venn==1.1.2 + + memoized-property==1.0.3 + + mhcflurry==2.2.1 + + mhcgnomes==3.33.5 + + mhcnames==0.4.8 + + mhctools==3.31.5 + + mpmath==1.3.0 + + msgspec==0.21.1 + + narwhals==2.25.0 + + networkx==3.6.1 + + numpy==2.4.6 + + oncoref==1.8.174 + + openpyxl==3.1.5 + + packaging==26.3 + + pandas==2.3.3 + + pdfkit==1.0.0 + + pillow==12.3.0 + + platformdirs==4.11.3 + + polars==1.43.2 + + polars-runtime-32==1.43.2 + + proglog==0.1.12 + + psutil==7.2.2 + + pyarrow==25.0.1 + + pycparser==3.0 + + pydyf==0.12.1 + + pyensembl==2.10.4 + + pyerfa==2.0.1.5 + + pyparsing==3.3.2 + + pyphen==0.18.1 + + pysam==0.24.0 + + python-codon-tables==0.1.18 + + python-dateutil==2.9.0.post0 + + pytz==2026.3.post1 + + pyyaml==6.0.3 + + requests==2.34.2 + + roman==5.2 + + scikit-learn==1.9.0 + + scipy==1.17.1 + + sercol==1.0.0 + + serializable==1.1.0 + + setuptools==84.0.0 + + simplejson==4.1.1 + + six==1.17.0 + + sympy==1.14.0 + + threadpoolctl==3.6.0 + + tinycss2==1.5.1 + + tinyhtml5==2.1.0 + + tinytimer==0.0.0 + + topiary==5.16.2 + + torch==2.13.0 + + tqdm==4.70.0 + + typechecks==0.1.0 + + typing-extensions==4.16.0 + + tzdata==2026.3 + + urllib3==2.7.0 + + varcode==5.0.6 + + vaxrank==3.1.14 (from file:///Users/steve/Documents/ChatGPT/NeoRepro%20%E2%80%94%20Fully%20Autonomous%20Research%20Project/predictors/vaxrank/source) + + weasyprint==69.0 + + webencodings==0.6.1 + + xvfbwrapper==0.2.29 + + zopfli==0.4.3 diff --git a/predictors/vaxrank/attempts/install.stdout.log b/predictors/vaxrank/attempts/install.stdout.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/vaxrank/attempts/smoke.stderr.log b/predictors/vaxrank/attempts/smoke.stderr.log new file mode 100644 index 0000000..e69de29 diff --git a/predictors/vaxrank/attempts/smoke.stdout.log b/predictors/vaxrank/attempts/smoke.stdout.log new file mode 100644 index 0000000..40a891b --- /dev/null +++ b/predictors/vaxrank/attempts/smoke.stdout.log @@ -0,0 +1 @@ +import ok diff --git a/predictors/vaxrank/attempts/sweep_receipt.json b/predictors/vaxrank/attempts/sweep_receipt.json new file mode 100644 index 0000000..3364844 --- /dev/null +++ b/predictors/vaxrank/attempts/sweep_receipt.json @@ -0,0 +1,12 @@ +{ + "predictor": "vaxrank", + "revision": "5c71edccda920fafaab5986837d3dd87c8ce8911", + "repo": "https://github.com/openvax/vaxrank.git", + "install_returncode": 0, + "smoke_returncode": 0, + "license_files": [ + "LICENSE" + ], + "platform": "macOS-15.3-arm64-arm-64bit", + "benchmark_track": "profile_only" +} diff --git a/reports/clean_reproduction.json b/reports/clean_reproduction.json index a5e9b7d..a67b847 100644 --- a/reports/clean_reproduction.json +++ b/reports/clean_reproduction.json @@ -1,10 +1,8 @@ { "schema_version": "1.0", - "source_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "command": "make -j4 reproduce-results", - "started_at": "2026-08-20T12:17:40Z", - "completed_at": "2026-08-20T12:29:54Z", - "elapsed_seconds": 734, + "source_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", + "command": "make -C /tmp/neorepro-release-hcLY3g -j4 reproduce-results", + "completed_at": "2026-08-21T07:53:30Z", "initial_worktree_clean": true, "returncode": 0, "runtime": { @@ -16,14 +14,20 @@ }, "verification": { "ruff": "pass", - "pytest": "27 passed", - "metric_validation": "24 checks passed at tolerance 1e-12", - "maximum_metric_absolute_error": 1.1102230246251565e-16, - "byte_stable_tracked_artifacts": true, - "post_run_changed_files_excluding_manifest": [] + "pytest": "49 passed", + "manuscript_audit": "pass; 12 citations resolved; no placeholders or forbidden overclaims", + "release_boundary": "pass; 413 tracked files; zero violations", + "metric_validation": "34 checks passed at tolerance 1e-12", + "maximum_metric_absolute_error": 2.220446049250313e-16, + "generated_outputs_promoted_from_clean_clone": true, + "post_run_findings": [ + "legacy generated CSV files used CRLF and were normalized to the writer's declared LF output", + "RCC metrics contained invocation-dependent absolute paths and were replaced by clean-clone relative paths" + ] }, "expected_manifest_change": { - "project_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "project_worktree_clean": true + "project_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", + "project_worktree_clean": false, + "reason": "the manifest is generated before the reproduction's final test step" } } diff --git a/reports/manuscript_resource_audit.json b/reports/manuscript_resource_audit.json index 36a5f18..d47e1f6 100644 --- a/reports/manuscript_resource_audit.json +++ b/reports/manuscript_resource_audit.json @@ -2,6 +2,7 @@ "citations_used": [ "albert2023bigmhc", "borch2024improve", + "braun2025rcc", "gfeller2023prime2", "kim2025tscape", "muller2023harmonized", diff --git a/reports/rcc_predictor_run.json b/reports/rcc_predictor_run.json new file mode 100644 index 0000000..ad82468 --- /dev/null +++ b/reports/rcc_predictor_run.json @@ -0,0 +1,108 @@ +{ + "artifacts": [ + { + "path": "results/raw_predictions/rcc/bigmhc-v1.0.csv", + "predictor": "BigMHC", + "rows": 129, + "sha256": "1b16e7bf545032a9763a9c899190775dddf24ea19378d8763899663a460f8dd5", + "version": "v1.0" + }, + { + "path": "results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "predictor": "DeepHLApan", + "rows": 129, + "sha256": "4d084a12522852fbdfe3bcbe9b64a8477f496b2073e6f8fc95943bfd18b3e4d4", + "version": "1.1.1@ac1f4beb" + }, + { + "path": "results/raw_predictions/rcc/deepimmuno-cnn.csv", + "predictor": "DeepImmuno-CNN", + "rows": 129, + "sha256": "be8fdb1f4efff6ef3159971a46f58513e8cc90d2d2f5737494aa14871700ad1e", + "version": "1.0@df42ac5b" + }, + { + "path": "results/raw_predictions/rcc/prime-2.0.csv", + "predictor": "PRIME", + "rows": 129, + "sha256": "68ed9f4f7a928964ccbab018d315e0e3ff9315d1b5c5a80665c30dee9a861140", + "version": "2.0" + } + ], + "benchmark": "data/processed/rcc_vaccine_benchmark.csv", + "completed_at": "2026-08-21T06:54:36.965390+00:00", + "orchestrator_python": "3.14.7 (main, Aug 7 2026, 02:15:30) [Clang 22.1.3 ]", + "parallel": true, + "platform": "macOS-15.3-arm64-arm-64bit-Mach-O", + "reuse_existing": false, + "runs": [ + { + "command": [ + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/.venv/bin/python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/source", + "--python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/bigmhc/.venv/bin/python", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/bigmhc-v1.0.csv" + ], + "execution": "executed", + "predictor": "bigmhc", + "returncode": 0, + "runtime_seconds": 65.49302654100029 + }, + { + "command": [ + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deephlapan/.venv/bin/python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deephlapan/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deephlapan/source", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deephlapan-1.1.1.csv" + ], + "execution": "executed", + "predictor": "deephlapan", + "returncode": 0, + "runtime_seconds": 51.63081233399862 + }, + { + "command": [ + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deepimmuno/.venv/bin/python", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deepimmuno/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/deepimmuno/source", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/deepimmuno-cnn.csv" + ], + "execution": "executed", + "predictor": "deepimmuno", + "returncode": 0, + "runtime_seconds": 12.130533249997825 + }, + { + "command": [ + "/Users/steve/.local/share/uv/python/cpython-3.14-macos-aarch64-none/bin/python3.14", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/prime/adapter.py", + "--input", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark.csv", + "--source-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/prime/source", + "--mix-dir", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/predictors/prime/vendor/mixmhcpred", + "--output", + "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/results/raw_predictions/rcc/prime-2.0.csv" + ], + "execution": "executed", + "predictor": "prime", + "returncode": 0, + "runtime_seconds": 2.97150474999944 + } + ] +} diff --git a/research/extension_protocol_rcc_v1.json b/research/extension_protocol_rcc_v1.json new file mode 100644 index 0000000..ffb5040 --- /dev/null +++ b/research/extension_protocol_rcc_v1.json @@ -0,0 +1,46 @@ +{ + "protocol_id": "neorepro-rcc-vaccine-extension-v1", + "frozen_at": "2026-08-20", + "status": "completed_prediction_and_evaluation", + "dataset": { + "name": "Braun et al. personalized RCC peptide vaccine cohort", + "article_doi": "10.1038/s41586-024-08507-5", + "supplement_member": "41586_2024_8507_MOESM4_ESM.xlsx", + "supplement_url": "https://media.springernature.com/original/springer-static/esm/art%3A10.1038%2Fs41586-024-08507-5/MediaObjects/41586_2024_8507_MOESM4_ESM.xlsx", + "supplement_sha256": "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1", + "worksheet": "sheet2", + "source_rows": 130, + "excluded_rows": 1, + "rows": 129, + "patients": 9, + "endpoint": "post-vaccination in-vitro peptide-stimulation IFN-gamma ELISpot", + "short_peptide_field": "Short_Epitope", + "hla_field": "HLA_of_best_short_epitope", + "patient_field": "Patient_ID", + "positive_rule": "source-reported individual-peptide p-value threshold; preserve continuous replicate values and source p-value", + "negative_rule": "not an untreated biological negative; row-level comparator is the matched no-stimulation control within the same assay", + "scope_boundary": "independent vaccine-elicited response domain; do not call untreated, natural presentation, clinical efficacy or intrinsic immunogenicity", + "eligibility_gate": [ + "129 retained rows have patient ID, short epitope and HLA field; one source row has N/A short epitope/HLA and is excluded", + "all 129 retained rows have three peptide-stimulation and three no-stimulation replicates", + "predicted HLA semantics are retained explicitly", + "no untested enumerated candidates are converted into negatives" + ] + }, + "prediction_policy": { + "status": "completed", + "eligible_tools": ["BigMHC v1.0", "PRIME 2.0", "DeepImmuno-CNN", "DeepHLApan"], + "exclude": ["MHCflurry from same-task primary because it is a presentation score"], + "primary": "patient-macro NDCG@5 on pairwise common prediction support", + "secondary": ["pooled AUROC", "average precision", "patient Recall@5", "coverage"], + "leakage": "apply known exact peptide-HLA union; unknown training identity remains unknown", + "reason_for_separate_protocol": "endpoint and treatment context differ from IMPROVE and Zhao", + "results": { + "BigMHC": {"predicted_rows": 128, "auroc": 0.476, "patient_ndcg5": 0.533}, + "PRIME": {"predicted_rows": 128, "auroc": 0.580, "patient_ndcg5": 0.691}, + "DeepHLApan": {"predicted_rows": 128, "auroc": 0.505, "patient_ndcg5": 0.614}, + "DeepImmuno-CNN": {"predicted_rows": 51, "auroc": 0.472, "patient_ndcg5": 0.711} + }, + "interpretation": "RCC is a small endpoint-distinct vaccine cohort; DeepImmuno-CNN has 39.5% coverage and all conclusions are descriptive, not universal model rankings." + } +} diff --git a/research/external_cohort_failure_protocols.json b/research/external_cohort_failure_protocols.json new file mode 100644 index 0000000..2eff722 --- /dev/null +++ b/research/external_cohort_failure_protocols.json @@ -0,0 +1,72 @@ +{ + "protocol_id": "neorepro-external-cohort-funnel-v1", + "created_at": "2026-08-20", + "status": "screening_complete_rcc_frozen_evx_failed", + "hard_gate": "Do not convert enumerated, untested, unreported, or merely computational candidates into experimental negatives.", + "screening_rule": [ + "Every retained row must be an individually assayed short peptide observation or a source-grounded explicit negative.", + "Patient identity, peptide, HLA and source study/cohort must be recoverable without guessing.", + "Treatment or vaccination context is recorded and cannot be relabeled untreated/natural presentation.", + "Training-set status is audited separately; unknown is not absence of overlap.", + "A source with incomplete evidence is pending, not eligible." + ], + "failures": [ + { + "candidate_id": "nci_mmp_2021", + "failure_code": "UNTESTED_ENUMERATED_CANDIDATES", + "evidence_url": "https://ndownloader.figshare.com/files/26040944", + "reason": "The archive enumerates short candidates derived from screened parent mutations, but does not establish individual testing for each short peptide-HLA row. Unknown rows cannot be labeled pMHC negatives.", + "action": "Preserve research/extension_protocol_v1_nci_failed.json; no prediction run." + }, + { + "candidate_id": "gbm_neovax_2025", + "failure_code": "NO_VERIFIED_COMPLETE_PEPTIDE_HLA_TABLE", + "evidence_url": "https://aacr.figshare.com/articles/dataset/Supplementary_Table_S5_from_Integrating_Multisector_Molecular_Characterization_into_Personalized_Peptide_Vaccine_Design_for_Patients_with_Newly_Diagnosed_Glioblastoma/29071987", + "reason": "Located public artifact is TCR-sequence data and does not itself verify complete patient-peptide-HLA experimental negatives.", + "action": "Do not download or infer missing rows; retain as failed until a source-grounded row-level table is found." + }, + { + "candidate_id": "melanoma_personal_2021", + "failure_code": "RESTRICTED_AND_ENDPOINT_INCOMPLETE", + "evidence_url": "https://pmc.ncbi.nlm.nih.gov/articles/PMC8273876/", + "reason": "Public supplements emphasize selected vaccine responses and the article states some patient-related data are subject to confidentiality/MTA; complete class-I patient-peptide-HLA experimental-negative rows are not established.", + "action": "Exclude from benchmark; do not treat unreported peptides as negatives." + }, + { + "candidate_id": "nepdb_2021", + "failure_code": "PROVENANCE_AND_TRAINING_UNCERTAIN", + "evidence_url": "https://doi.org/10.3389/fimmu.2021.644637", + "reason": "A curated database is not automatically a reconstructable patient-grouped cohort; patient IDs, assay denominators, cohort boundaries and training overlap are not verified at row level.", + "action": "Use only for discovery/provenance cross-checking, not as an external cohort." + }, + { + "candidate_id": "tesla_2020", + "failure_code": "KNOWN_TRAINING_OVERLAP", + "evidence_url": "https://doi.org/10.1016/j.cell.2020.09.015", + "reason": "The local audit found all 520 pilot rows exact peptide-HLA matches in official PRIME2 training and concordant with the published BigMHC immunogenicity construction.", + "action": "Retain only as leakage-positive control; no external-validity claim." + }, + { + "candidate_id": "itsndb_2023", + "failure_code": "PATIENT_AND_NEGATIVE_SEMANTICS_NOT_VERIFIED", + "evidence_url": "https://doi.org/10.3389/fimmu.2023.1094236", + "reason": "The public description mixes curated neoantigens with simulated prioritization scenarios; a complete patient-linked, individually tested negative table is not established.", + "action": "Do not promote to candidate without source-table audit." + }, + { + "candidate_id": "evx01_2024", + "failure_code": "NO_COMPLETE_SHORT_PEPTIDE_HLA_NEGATIVE_TABLE", + "evidence_url": "https://jitc.bmj.com/content/12/5/e008817", + "reason": "The study reports individual restimulation of long vaccine peptides, but the public evidence does not provide a complete short-peptide-HLA row table with experimentally defined pMHC negatives; irrelevant-peptide background is not equivalent to a source-grounded pMHC negative cohort.", + "action": "Exclude from benchmark and preserve the source-level failure." + } + ], + "pending": [], + "eligible_after_member_audit": [ + { + "candidate_id": "rcc_pcv_2025", + "protocol": "research/extension_protocol_rcc_v1.json", + "member_sha256": "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1" + } + ] +} diff --git a/research/external_cohort_funnel.csv b/research/external_cohort_funnel.csv new file mode 100644 index 0000000..4fd02b3 --- /dev/null +++ b/research/external_cohort_funnel.csv @@ -0,0 +1,11 @@ +candidate_id,title,year,source_url,source_type,individual_short_peptide_assay,patient_id,hla,experimental_negative,study_cohort_recoverable,redistribution,training_set_risk,eligibility,decision,evidence_note,accessed_at +improve_2024,IMPROVE feature model broad-scale validation of T-cell recognition,2024,https://doi.org/10.3389/fimmu.2024.1360281,paper+repository,yes,yes,yes,yes,yes,public repository terms require audit,unknown,eligible,existing_primary,"17,520 patient-matched screened records; already frozen as principal benchmark; exact PRIME2 overlap audited separately.",2026-08-20 +zhao_2026,Profiling immunogenic neoantigen peptides elicited by personalized neoantigen vaccine,2026,https://pmc.ncbi.nlm.nih.gov/articles/PMC13286890/,paper+Europe PMC supplement,yes,yes,yes,yes,yes,CC BY 4.0,partial,eligible,existing_external,"2,317 administered 8-11mer peptides from 352 patients; individual IFN-gamma ELISPOT; 313 source positives under ratio >=2.0; already frozen and run.",2026-08-20 +nci_mmp_2021,NCI Surgery Branch mutated minimal peptide test set,2021,https://ndownloader.figshare.com/files/26040944,Figshare archive,unknown,partial,yes,no,yes,CC BY 4.0,unknown,failed,failed_eligibility,"2.36-GB file enumerates short candidates derived from parent screens; row-level short-peptide experimental testing is not established, so unknown rows cannot be negatives.",2026-08-20 +rcc_pcv_2025,A neoantigen vaccine generates antitumour immunity in renal cell carcinoma,2025,https://doi.org/10.1038/s41586-024-08507-5,Nature supplement,yes,yes,yes,partial,yes,partial,CC BY-NC-ND 4.0,eligible,eligible_external,"Checksum-pinned Supplementary Table 2 sheet 2 has 130 source rows from 9 patients; 129 retained rows have canonical short epitope, predicted HLA and three peptide-stimulation plus three no-stimulation replicates. One N/A short-epitope/HLA row is excluded. Positives are defined by source p-value threshold; endpoint is post-vaccine ELISpot and therefore endpoint-distinct from untreated natural presentation.",2026-08-20 +evx01_2024,Dose escalation study of a personalized peptide-based neoantigen vaccine,2024,https://jitc.bmj.com/content/12/5/e008817,Nature/PMC supplement,yes,yes,partial,partial,yes,partial,CC BY-NC,failed,failed_eligibility,"The paper reports 91 individually restimulated long vaccine peptides and patient IDs, but the public evidence does not expose a complete row-level short-peptide-HLA table with experimental negatives; HLA/minimal epitopes are prediction-derived and responses use irrelevant-peptide background rather than a verified pMHC negative table.",2026-08-20 +gbm_neovax_2025,Integrating Multisector Molecular Characterization into Personalized Peptide Vaccine Design for GBM,2025,https://aacr.figshare.com/articles/dataset/Supplementary_Table_S5_from_Integrating_Multisector_Molecular_Characterization_into_Personalized_Peptide_Vaccine_Design_for_Patients_with_Newly_Diagnosed_Glioblastoma/29071987,Figshare supplement,partial,yes,unknown,partial,yes,CC BY 4.0,unknown,failed,failed_eligibility,"Figshare item is TCR-sequence data; article says IFN-gamma ELISPOT for patients 1-3, but the located artifact does not establish a complete row-level pMHC table with HLA and experimental negatives.",2026-08-20 +melanoma_personal_2021,Personal neoantigen vaccines induce persistent memory T-cell responses,2021,https://pmc.ncbi.nlm.nih.gov/articles/PMC8273876/,paper+supplement,partial,yes,partial,no,yes,restricted/MTA for some data,unknown,failed,failed_eligibility,"Supplementary data emphasize vaccine-positive responses, pools and class-II work; paper states patient-related data may require MTA and does not establish a complete individually tested class-I negative table.",2026-08-20 +nepdb_2021,NEPdb experimentally validated neoantigens and predicted neoepitopes,2021,https://doi.org/10.3389/fimmu.2021.644637,database,partial,unknown,yes,partial,partial,public site terms unclear,high,failed,failed_eligibility,"Curated records include effective/ineffective entries and HLA, but patient identity, assay denominator and source-cohort reconstruction are not guaranteed at record level; database/model-training overlap risk is high.",2026-08-20 +tesla_2020,TESLA consortium validation set,2020,https://doi.org/10.1016/j.cell.2020.09.015,paper+supplement,yes,yes,yes,yes,yes,source-specific terms,known,failed,failed_leakage_gate,"Experimentally tested patient-matched records are real, but all 520 retained pilot rows exactly overlap PRIME2 training and BigMHC immunogenicity construction; retained only as leakage-positive control.",2026-08-20 +itsndb_2023,Unraveling tumor-specific neoantigen immunogenicity prediction,2023,https://doi.org/10.3389/fimmu.2023.1094236,paper+database,partial,partial,yes,partial,partial,unknown,high,failed,failed_eligibility,"Includes curated presented neoantigens and simulated prioritization scenarios, but the full patient-linked experimental-negative contract needed here is not established from the public article/database description.",2026-08-20 diff --git a/research/research_log.md b/research/research_log.md index bd313e2..8ab60f8 100644 --- a/research/research_log.md +++ b/research/research_log.md @@ -87,3 +87,38 @@ - The Zhao source yielded 2,317 individually administered peptides from 352 patients (313 positives). Known exact training-overlap union exclusion removed two positive records; 2,315 records, 311 positives and 131 positive-bearing patients remained. - The frozen 2,000-replicate primary comparison found BigMHC above PRIME by 0.057 patient-macro NDCG@5 (95% CI 0.008–0.106) and above DeepHLApan by 0.078 (0.022–0.133) on near-complete common support. DeepImmuno-CNN covered 43.8%; its common-support differences were unresolved. - The BigMHC–PRIME direction reversed relative to IMPROVE. This is interpreted as model-by-domain dependence, not a universal-winner result. The vaccine-elicited post-vaccination ELISPOT endpoint remains distinct from natural tumor presentation and clinical benefit. + +## 2026-08-20 — External-cohort funnel audit (third/fourth queue) + +- Added `research/external_cohort_funnel.csv` and `research/external_cohort_failure_protocols.json` to preserve a source-by-source eligibility funnel rather than treating literature mentions as datasets. +- Screened ten candidates spanning IMPROVE, Zhao, NCI, RCC PCV, EVX-01, GBM NeoVax, melanoma personal vaccines, NEPdb, TESLA and ITSNdb. Zhao and IMPROVE remain the only currently verified eligible patient-grouped benchmark datasets in this repository; Zhao has already been frozen and run. +- NCI, GBM NeoVax, melanoma personal-vaccine data, NEPdb and ITSNdb received explicit failure reasons. NCI remains the decisive example of the mandatory rule: enumerated or untested short candidates are not experimental negatives. +- RCC PCV and EVX-01 remain `pending` rather than eligible because the public evidence establishes individual-peptide assay components but does not yet verify the complete patient–peptide–HLA row-level negative contract, exact HLA semantics and machine-readable redistribution path. +- No new predictions were launched from a pending source. The next safe action is a targeted supplement-member audit followed by a new frozen protocol only if every eligibility field passes. + +## 2026-08-20 — RCC member audit and EVX-01 exclusion + +- RCC Supplementary Table 2 was downloaded from the official Nature member URL and checksum-pinned (`c113c42b...5d0c1`). Worksheet 2 contains 130 rows from 9 patients; every row has a short epitope, predicted HLA field, three peptide-stimulation replicates and three no-stimulation replicates. RCC is therefore eligible as an endpoint-distinct vaccine cohort, and `research/extension_protocol_rcc_v1.json` was frozen before prediction. +- RCC is not untreated and its HLA values are predicted binding alleles; all downstream reporting must preserve those limitations. It cannot be merged with IMPROVE as natural presentation or with Zhao as an identical assay endpoint. +- EVX-01 was excluded after source-level audit. The paper reports 91 individually restimulated long vaccine peptides, but the public evidence does not provide a complete short-peptide-HLA row table with experimentally defined pMHC negatives; its background is an irrelevant peptide control. No prediction was run for EVX-01. + +## 2026-08-20 — RCC extension prediction and evaluation completed + +- Built `data/processed/rcc_vaccine_benchmark.csv` from 129 retained rows after excluding one N/A short-epitope/HLA source row; 9 patients, 75 positives and 54 negatives. Source checksum and row-level provenance are retained in `data/rcc_vaccine_summary.json` and the canonical rows. +- Training-overlap audit found zero exact PRIME2, published BigMHC construction, or DeepImmuno peptide-HLA overlaps; near-sequence results are retained, and DeepHLApan training identity remains unknown. +- Fixed predictors ran with coverage: BigMHC 128/129, PRIME 128/129, DeepHLApan 128/129, DeepImmuno-CNN 51/129. Missing outputs were not imputed or treated as negatives. +- Existing evaluation produced descriptive pooled AUROC / patient NDCG@5: BigMHC 0.476 / 0.533, PRIME 0.580 / 0.691, DeepHLApan 0.505 / 0.614, DeepImmuno-CNN 0.472 / 0.711. Unequal support, vaccine endpoint, predicted-HLA semantics and small patient count prohibit universal ranking claims. + +## 2026-08-20 — Public predictor reproduction sweep + +- Added `scripts/reproduce_public_predictors.py` to clone each queued public repository, capture the immutable HEAD revision and license-file paths, create a per-predictor Python 3.11 environment, run editable installation, and preserve install/smoke stdout, stderr, and JSON receipts. +- MHCnuggets, NeoFox, pVACtools, Vaxrank, and mhcmatch installed successfully. MHCnuggets and NeoFox passed import smoke tests; pVACtools and Vaxrank passed `--help`. mhcmatch has no documented smoke entry point in the checked revision. +- NeoGuider did not expose a Python package entry point (`pyproject.toml`/`setup.py` absent), and Seq2Neo requires TensorFlow 2.3.0, which has no Python 3.11 wheel. These are recorded as reproducibility failures, not silently repaired. +- pVACtools, NeoFox, Vaxrank, and Seq2Neo remain profile-only or non-comparable because their output contracts are end-to-end/annotation/ranking workflows rather than the canonical peptide–HLA score contract. No new candidate entered the benchmark. +- Follow-up MHCnuggets audit: production BA weights and curated training tables are present in the pinned repository. The official three-peptide example ran successfully under Python 3.11/TensorFlow and the strict adapter produced 3/3 binding scores with lower-IC50-is-better semantics. The repository's `saves/test/HLA-A01:01_test_model` is a separate test checkpoint whose expected values cannot validate the production checkpoint; this is recorded as a limitation rather than an exact-match claim. The predictor remains pending for binding-track inclusion until training-overlap and benchmark-task alignment are audited. +- Efficiency correction: mhcmatch's first binder invocation triggered its expensive calibration/reference bootstrap and timed out at 180 seconds. The cheap local CLI/API smoke path (`--help`, `decompose`) passed; the timeout is retained as evidence of an external/reference-data dependency, and no further large bootstrap was attempted for this profile-only candidate. +## 2026-08-20 — Cross-dataset stability and model-selection-risk analysis + +Implemented `scripts/analyze_stability.py` and generated `results/analysis/stability/` from the frozen IMPROVE and Zhao benchmark/prediction artifacts. The analysis is explicitly exploratory/descriptive heterogeneity analysis: it produces a dataset × predictor × metric matrix, record-level Spearman ranking concordance, patient-bootstrap probability of being first at Recall@5, BigMHC–PRIME direction-reversal probability, and coverage-threshold common-support summaries. It uses analytic fixed-score rankings and a recorded seed; it does not perform post-hoc significance testing, causal inference, or clinical efficacy claims. The script preserves predictor status metadata and writes `analysis_metadata.json` with limitations and configuration. + +Acceptance follow-up: added explicit `leave_one_domain_out.csv`, `endpoint_domain_metadata.csv`, a dependency-free endpoint/domain AUROC SVG, and `tests/test_stability_analysis.py`. First-place probabilities are task-stratified so presentation and immunogenicity predictors are not placed in one leaderboard. The expensive 2,000-bootstrap output remains the authoritative model-selection file; the leave-one-domain-out and visualization artifacts are deterministic post-processing of the frozen inputs. diff --git a/research/training_overlap_audit_rcc.csv b/research/training_overlap_audit_rcc.csv new file mode 100644 index 0000000..5ae946c --- /dev/null +++ b/research/training_overlap_audit_rcc.csv @@ -0,0 +1,130 @@ +record_id,benchmark_label,exact_prime2_peptide_hla,exact_bigmhc_im_trainval,exact_deepimmuno_peptide_hla,union_known_exact_overlap,prime2_label_conflict,deepimmuno_label_conflict,peptide_only_prime2_different_hla,peptide_only_deepimmuno_different_hla,near_hamming1_prime2_same_hla,near_hamming1_deepimmuno_same_hla,deephlapan_training_overlap +rcc-vaccine-8c2e1ec6d8f675cf,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6a6ffb8b81ec9208,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4adeedbe0e35cf09,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e06799bdf8de8af5,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-969998ab9d8b4dc1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a5952e9e71bd27bd,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6fa3e9ad450e030e,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-8a9041cc55b08e38,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e11c81feedc5751e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-fefffdcccca8649c,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-940b62df9c3078c4,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-43742d14e082c2e6,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-769c6b305827c570,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e74f183589095f61,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-15ab70cb934aad8f,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c8a8ce5ab15f9d4b,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f05602883865f4bf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b92105a12fb6f028,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a004075aed453bcf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-70eb710f90e2973b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c81528933be4a147,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-2a2f0b8f959aa3f6,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-74e8a99b7ef86c61,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-88bea2527dd72a1d,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ff6bbb8d69468a47,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-8ea6c4f21d2c81ee,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b2f6110d808ac0da,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f43e93564087f160,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-fc1276f53c3f45e1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-3e6b1c1d8de0ce71,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4255bdded9e984bd,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-31c3a5a1c962db43,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-d3d4046d155047b6,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-5e4096b57e93c92e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e8d6aeb51705642a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-26dc24e186221fe2,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-364822c022710958,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9c477d13a3359f5e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-487a8e43d12c17ac,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-539b50b223ea0ee1,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-618d6aa7d8232657,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-71f5d2ab19a389e8,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-409a0661a319bbb1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9f431d040990f0f7,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7c1009a0bb0524b3,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-42e869aba90dc4bd,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a1a88d1c11b4fcdd,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-d42b9fb3a44366ac,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ce806d8b1a4132b2,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ea51c18cc4e16995,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9bce007ba3a4aff0,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-aa3019acbdddb7dc,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-1bcdaed331509a08,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4a1853322eea7a14,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-cde49959d75e6ec2,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7f8e1f8e02332663,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b8aa4d97d214f765,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-5993797f7a6c7057,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-eacb2f310488ca9d,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-70f05fcfa038224f,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4d15c4e62bb19311,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a6b2180415ff1b56,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e52f3609d694b4ba,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-763f29ff7ac4c65d,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c331f65899396b9b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-09c9b699d977f421,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e944d5a782a7d074,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-75f86cf28dca60e2,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-5b2ef450eaf2248a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c50e7f0e1b3b69a5,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ae70f68909b113e8,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-efdf21635c142ab8,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-817b1ac6328997a7,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-feb66d3c66c8a4de,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7946e9a10fe114cf,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f3e32e857dd09d71,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-240bf5b232815c6a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-40c01a17a3ed4a2e,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7b823cc728ea1450,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-0c7eb5e3295f0ad2,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-88aaf2311ce63196,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-bcae618cfad77c1f,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-11f61d47947266cc,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f4a4db2362e7a54e,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-1eceb5b6a969d721,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-55106745d24cfbf4,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a808947689fbe42c,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-590f5ea08f24e2cf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b984dc3be5c1fa17,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6ff6e758ac279425,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-aa0d4e0503dacba4,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4d44829f31cf5558,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4f7d9f9f22d48440,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-60d1ba1b8ae350fd,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7ec85836cb09ee30,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-4b7fdcc54f65b98b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-d02de725ecc685a5,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e1d984ebf245728f,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-a9a3c2bc98a74746,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-da610226e338a8de,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ca0a8a8bb4e2d094,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-c45585f253341070,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b2761351ccb757b7,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b1ad936ccc2f249a,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-02261377bb800592,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-834aff6d2a7dc363,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-0036b8c45a4cd20d,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-08447808dd0300dd,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-34c3e7e7a2dcd8b6,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-1ae968c5ae01f49b,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-29f7c92d094c1ced,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-0aaef5904890371c,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-27b47d43dab2b353,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-46b486b9d199e93a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-b8a9c44bcf53568f,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-7425139a6230adc0,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e39c49c350f42009,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-73717d4619b66faf,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-f130cbb5ea1ca0fa,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-3f16099c911d2791,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-07bc8054e1edd65a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-99bd603a759d6b22,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-ad726d8ef91bd0e6,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-972e1f28fe4cb96a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-e0d628668c1eccf2,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-3d165d772065bb3a,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-9ef49682993d2ea1,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-6c01a27a6a575007,1,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable +rcc-vaccine-12b152d35907e280,0,0,0,0,0,0,0,0,0,0,0,unknown_official_row_manifest_unavailable diff --git a/research/training_overlap_summary_rcc.json b/research/training_overlap_summary_rcc.json new file mode 100644 index 0000000..89e2c02 --- /dev/null +++ b/research/training_overlap_summary_rcc.json @@ -0,0 +1,23 @@ +{ + "benchmark": "/Users/steve/Documents/ChatGPT/NeoRepro \u2014 Fully Autonomous Research Project/data/processed/rcc_vaccine_benchmark_full.csv", + "benchmark_exact_bigmhc_im_trainval": 0, + "benchmark_exact_deepimmuno": 0, + "benchmark_exact_prime2": 0, + "benchmark_near_deepimmuno": 0, + "benchmark_near_prime2": 0, + "benchmark_rows": 129, + "benchmark_union_known_exact_overlap": 0, + "deepimmuno_training_rows_resolved": 9055, + "deepimmuno_training_rows_source": 9055, + "deepimmuno_unresolved_hla_rows": 0, + "overlap_dimensions": { + "bigmhc_published_immunogenicity_construction": "checked_via_prime2_random_flag", + "deephlapan_training_identity": "unknown_official_row_manifest_unavailable", + "deepimmuno_exact_peptide_hla": "checked", + "near_sequence_same_hla_same_length": "checked_for_prime2_and_deepimmuno", + "patient_and_study_training_identity": "unavailable", + "prime2_exact_peptide_hla": "checked" + }, + "prime2_training_rows": 65585, + "prime2_workbook_sha256": "641a104764167f9f04bafb6606e519e5625740ed1720af7d15ac9026636bc23a" +} diff --git a/results/analysis/improve/baselines/lopo/missingness.csv b/results/analysis/improve/baselines/lopo/missingness.csv index c1c47e3..1e9c7fc 100644 --- a/results/analysis/improve/baselines/lopo/missingness.csv +++ b/results/analysis/improve/baselines/lopo/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -HLA-only LR LOPO,predicted,17475 -HLA+peptide LR LOPO,predicted,17475 -Peptide LR LOPO,predicted,17475 +predictor,status,count +HLA-only LR LOPO,predicted,17475 +HLA+peptide LR LOPO,predicted,17475 +Peptide LR LOPO,predicted,17475 diff --git a/results/analysis/improve/baselines/lopo/paired_differences.csv b/results/analysis/improve/baselines/lopo/paired_differences.csv index 8b50859..35e705f 100644 --- a/results/analysis/improve/baselines/lopo/paired_differences.csv +++ b/results/analysis/improve/baselines/lopo/paired_differences.csv @@ -1,46 +1,46 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.562322289861119,0.07206248064073639,0.0432272336198909,0.10077865151752015 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.03802950148325792,0.010817584063849035,0.0013040512335336714,0.021318739789603722 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.3241702939840256,0.2591630393493078,0.1561631429974733,0.3661550244802481 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.4962937058813608,0.17037296078530584,0.08526341930256828,0.2613709238066563 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.19168103601308425,0.1249856306535824,0.02573337786382624,0.24086538865079202 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.13187818435593066,0.12544883588421643,0.053615353704524124,0.20699299010461514 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.05544322396267425,0.0571610770978802,0.02919128458541056,0.08672210872874804 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.07953454882823269,0.06339437108942386,0.0330266870021765,0.0974934892604656 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.046264237753205706,0.04948447393881804,0.01577981645909171,0.0867659236192426 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.04431359387664594,0.040686406123354064,0.021745338836508022,0.06092781505166672 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.044919654482706545,0.026747012183960125,0.01282387235798424,0.041761184872791436 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.04431359387664594,0.0390197394566874,0.010404583887577357,0.07079106246187 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.05915812678980218,0.05431651542029429,0.02095443039701796,0.09091761560448085 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.1213465566099074,0.08449070697256542,0.0329379099352948,0.14927716608616526 -HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.02957906339490109,0.018820778554169536,-0.00011042603165881617,0.04106000911749209 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.6283226185735147,0.006062151928340698,-0.02726384106308529,0.03987202282874768 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.046875818599155246,0.0019712669479517095,-0.008447824496845195,0.011326186671004641 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.45,0.13333333333333336,0.0,0.27122496147919856 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.6333333333333333,0.033333333333333326,-0.10344827586206895,0.15877976190476165 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.31666666666666665,0.0,-0.10772727272727274,0.1016949152542373 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.25012340938813493,0.0072036108520121545,-0.07287285962184244,0.08286646815910921 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.11701709766513434,-0.004412796604579883,-0.038038561847143756,0.028067246340345093 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.1459883779160683,-0.0030594579984117576,-0.041580528827991056,0.03469832763901281 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.10602336890466177,-0.010274657212638025,-0.048478099238820044,0.0274142508981307 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.083,0.0020000000000000018,-0.016949152542372885,0.020312499999999997 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.07229166666666667,-0.0006250000000000006,-0.01631555640455682,0.015873436238902344 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.09166666666666667,-0.008333333333333331,-0.04091095380029806,0.0230833333333333 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.11818160611644385,-0.004706963906347381,-0.0591545174571619,0.042768502687826025 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.21154286614806259,-0.0057056025655897735,-0.0728350385466207,0.057860254892542234 -HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.05441761648224497,-0.0060177745331743415,-0.032539747818009876,0.01553704952974984 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.562322289861119,0.6283226185735147,-0.0660003287123957,-0.11832890982241072,-0.013342082465851564 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.03802950148325792,0.046875818599155246,-0.008846317115897326,-0.023381279249302375,0.005429276260053588 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.3241702939840256,0.45,-0.12582970601597443,-0.2732621449104854,0.017708865829779225 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.4962937058813608,0.6333333333333333,-0.1370396274519725,-0.27083766623619915,-0.00424252670827134 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.19168103601308425,0.31666666666666665,-0.1249856306535824,-0.2528735344590615,-0.002628648721055054 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.13187818435593066,0.25012340938813493,-0.11824522503220428,-0.2111754582389387,-0.0315827093793508 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.05544322396267425,0.11701709766513434,-0.061573873702460084,-0.10753137077430959,-0.02009169118529757 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.07953454882823269,0.1459883779160683,-0.06645382908783562,-0.11709273407121512,-0.022840850217881428 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.046264237753205706,0.10602336890466177,-0.05975913115145606,-0.10841227836082305,-0.016267059525814625 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.04431359387664594,0.083,-0.03868640612335406,-0.06675304151403763,-0.010740974429773268 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.044919654482706545,0.07229166666666667,-0.027372012183960126,-0.05052328585231764,-0.005996993293902756 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.04431359387664594,0.09166666666666667,-0.04735307279002073,-0.08744525528113761,-0.011389367329959632 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.05915812678980218,0.11818160611644385,-0.05902347932664167,-0.11463921312886032,-0.010987650944993977 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.1213465566099074,0.21154286614806259,-0.09019630953815519,-0.1667874400316495,-0.020624254502470137 -HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.02957906339490109,0.05441761648224497,-0.024838553087343878,-0.05522741371521793,0.0014403446508246328 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.562322289861119,0.07206248064073639,0.0432272336198909,0.10077865151752015 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.03802950148325792,0.010817584063849035,0.0013040512335336714,0.021318739789603722 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.3241702939840256,0.2591630393493078,0.1561631429974733,0.3661550244802481 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.4962937058813608,0.17037296078530584,0.08526341930256828,0.2613709238066563 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.19168103601308425,0.1249856306535824,0.02573337786382624,0.24086538865079202 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.13187818435593066,0.12544883588421643,0.053615353704524124,0.20699299010461514 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.05544322396267425,0.0571610770978802,0.02919128458541056,0.08672210872874804 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.07953454882823269,0.06339437108942386,0.0330266870021765,0.0974934892604656 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.046264237753205706,0.04948447393881804,0.01577981645909171,0.0867659236192426 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.04431359387664594,0.040686406123354064,0.021745338836508022,0.06092781505166672 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.044919654482706545,0.026747012183960125,0.01282387235798424,0.041761184872791436 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.04431359387664594,0.0390197394566874,0.010404583887577357,0.07079106246187 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.05915812678980218,0.05431651542029429,0.02095443039701796,0.09091761560448085 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.1213465566099074,0.08449070697256542,0.0329379099352948,0.14927716608616526 +HLA+peptide LR LOPO,HLA-only LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.02957906339490109,0.018820778554169536,-0.00011042603165881617,0.04106000911749209 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.6343847705018554,0.6283226185735147,0.006062151928340698,-0.02726384106308529,0.03987202282874768 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.048847085547106955,0.046875818599155246,0.0019712669479517095,-0.008447824496845195,0.011326186671004641 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.5833333333333334,0.45,0.13333333333333336,0.0,0.27122496147919856 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.6666666666666666,0.6333333333333333,0.033333333333333326,-0.10344827586206895,0.15877976190476165 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.31666666666666665,0.31666666666666665,0.0,-0.10772727272727274,0.1016949152542373 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.2573270202401471,0.25012340938813493,0.0072036108520121545,-0.07287285962184244,0.08286646815910921 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.11260430106055445,0.11701709766513434,-0.004412796604579883,-0.038038561847143756,0.028067246340345093 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.14292891991765655,0.1459883779160683,-0.0030594579984117576,-0.041580528827991056,0.03469832763901281 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.09574871169202374,0.10602336890466177,-0.010274657212638025,-0.048478099238820044,0.0274142508981307 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.085,0.083,0.0020000000000000018,-0.016949152542372885,0.020312499999999997 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.07166666666666667,0.07229166666666667,-0.0006250000000000006,-0.01631555640455682,0.015873436238902344 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.08333333333333334,0.09166666666666667,-0.008333333333333331,-0.04091095380029806,0.0230833333333333 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.11347464221009647,0.11818160611644385,-0.004706963906347381,-0.0591545174571619,0.042768502687826025 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.2058372635824728,0.21154286614806259,-0.0057056025655897735,-0.0728350385466207,0.057860254892542234 +HLA+peptide LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.04839984194907063,0.05441761648224497,-0.0060177745331743415,-0.032539747818009876,0.01553704952974984 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,auroc,17475,465,70,0.562322289861119,0.6283226185735147,-0.0660003287123957,-0.11832890982241072,-0.013342082465851564 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,average_precision,17475,465,70,0.03802950148325792,0.046875818599155246,-0.008846317115897326,-0.023381279249302375,0.005429276260053588 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@10,17475,465,70,0.3241702939840256,0.45,-0.12582970601597443,-0.2732621449104854,0.017708865829779225 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@20,17475,465,70,0.4962937058813608,0.6333333333333333,-0.1370396274519725,-0.27083766623619915,-0.00424252670827134 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,hitrate@5,17475,465,70,0.19168103601308425,0.31666666666666665,-0.1249856306535824,-0.2528735344590615,-0.002628648721055054 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,mrr,17475,465,70,0.13187818435593066,0.25012340938813493,-0.11824522503220428,-0.2111754582389387,-0.0315827093793508 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@10,17475,465,70,0.05544322396267425,0.11701709766513434,-0.061573873702460084,-0.10753137077430959,-0.02009169118529757 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@20,17475,465,70,0.07953454882823269,0.1459883779160683,-0.06645382908783562,-0.11709273407121512,-0.022840850217881428 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,ndcg@5,17475,465,70,0.046264237753205706,0.10602336890466177,-0.05975913115145606,-0.10841227836082305,-0.016267059525814625 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@10,17475,465,70,0.04431359387664594,0.083,-0.03868640612335406,-0.06675304151403763,-0.010740974429773268 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@20,17475,465,70,0.044919654482706545,0.07229166666666667,-0.027372012183960126,-0.05052328585231764,-0.005996993293902756 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,precision@5,17475,465,70,0.04431359387664594,0.09166666666666667,-0.04735307279002073,-0.08744525528113761,-0.011389367329959632 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@10,17475,465,70,0.05915812678980218,0.11818160611644385,-0.05902347932664167,-0.11463921312886032,-0.010987650944993977 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@20,17475,465,70,0.1213465566099074,0.21154286614806259,-0.09019630953815519,-0.1667874400316495,-0.020624254502470137 +HLA-only LR LOPO,Peptide LR LOPO,immunogenicity_lopo,recall@5,17475,465,70,0.02957906339490109,0.05441761648224497,-0.024838553087343878,-0.05522741371521793,0.0014403446508246328 diff --git a/results/analysis/improve/baselines/loso/missingness.csv b/results/analysis/improve/baselines/loso/missingness.csv index ecd3c4a..e902c54 100644 --- a/results/analysis/improve/baselines/loso/missingness.csv +++ b/results/analysis/improve/baselines/loso/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -HLA-only LR LOSO,predicted,17475 -HLA+peptide LR LOSO,predicted,17475 -Peptide LR LOSO,predicted,17475 +predictor,status,count +HLA-only LR LOSO,predicted,17475 +HLA+peptide LR LOSO,predicted,17475 +Peptide LR LOSO,predicted,17475 diff --git a/results/analysis/improve/baselines/loso/paired_differences.csv b/results/analysis/improve/baselines/loso/paired_differences.csv index 28f1c55..e13db62 100644 --- a/results/analysis/improve/baselines/loso/paired_differences.csv +++ b/results/analysis/improve/baselines/loso/paired_differences.csv @@ -1,46 +1,46 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.5459065192518,0.048947804264411254,0.020457464895754698,0.07937486042024944 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.032614680056597,0.010077252856861799,0.0031023328080865167,0.020747831469716746 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.2979086632025005,0.1687580034641662,0.07155554235306978,0.2774480293319457 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.47210862793233316,0.14455803873433354,0.039983410998719326,0.24345803450121287 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.16997512612276583,0.18002487387723415,0.07716133175732295,0.28800506073439314 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.12138679992279434,0.13662641968612688,0.06263029181255392,0.21962079097314244 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.0509966130685172,0.055910582846923976,0.02846771961923004,0.08615472760019419 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.07317468265746373,0.06008822497603074,0.03135873418626387,0.09056310898748074 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.04159745451771191,0.05832264603511714,0.025782275188904233,0.09329060029745202 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.03803862528134434,0.04112804138532232,0.020686739564114716,0.06401784999313889 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.03808248493046715,0.032750848402866184,0.015999736614523856,0.050405995011715395 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.03803862528134434,0.048628041385322326,0.0213051106695626,0.0783005935087021 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.05961413544748199,0.040723161483708394,0.01119386649032003,0.07329717629672271 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.11937446972537334,0.05250218823287424,0.007045304398280664,0.0969215226731168 -HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.029807067723740994,0.02513657764413305,0.0029390636108198695,0.04945687861432687 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.6188880671078999,-0.02403374359168864,-0.06339992272149965,0.015856466810683638 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.04488912649824684,-0.0021971935847880444,-0.010770780756085853,0.008134134304698228 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.45,0.016666666666666663,-0.09836065573770492,0.1273167155425218 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.5833333333333334,0.033333333333333326,-0.08932629870129873,0.15517241379310343 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.2916666666666667,0.05833333333333329,-0.03448275862068961,0.15384615384615383 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.2060379930533435,0.05197522655557771,-0.024532424987950337,0.1282996253126007 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.10329449749236344,0.003612698423077737,-0.025988054059977353,0.035371420451524464 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.13007923122688464,0.0031836764066098255,-0.031152731601539863,0.037255786595359934 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.08624215640883011,0.013677944143998935,-0.01777065475939925,0.045118767142931934 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.08633333333333333,-0.007166666666666668,-0.027499999999999997,0.015695454545454523 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.07083333333333333,0.0,-0.013890027322404365,0.01428571428571429 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.08222222222222222,0.004444444444444445,-0.023503056404556823,0.03114754098360656 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.10534099765083538,-0.005003700719644996,-0.04396245631017302,0.03553267380973812 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.19087821976360195,-0.019001561805354367,-0.08353079239282275,0.042516123849899154 -HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.04305045836052295,0.011893187007351091,-0.009024639249381839,0.035618348876926255 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5459065192518,0.6188880671078999,-0.0729815478560999,-0.12875313741264985,-0.015537675879811655 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.032614680056597,0.04488912649824684,-0.012274446441649843,-0.024341311717841006,-0.00250709977379717 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.2979086632025005,0.45,-0.15209133679749953,-0.28980508854756826,-0.021391401117801596 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.47210862793233316,0.5833333333333334,-0.11122470540100021,-0.2410105261552087,0.021158512901603768 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.16997512612276583,0.2916666666666667,-0.12169154054390086,-0.23840263822057875,-0.016857306346675458 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.12138679992279434,0.2060379930533435,-0.08465119313054915,-0.1682251905117228,-0.013927669926753318 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.0509966130685172,0.10329449749236344,-0.05229788442384624,-0.09277793584475871,-0.014818739588729708 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.07317468265746373,0.13007923122688464,-0.05690454856942091,-0.10314165793053343,-0.013756379356776267 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.04159745451771191,0.08624215640883011,-0.0446447018911182,-0.08615668756449882,-0.00797460488548041 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.03803862528134434,0.08633333333333333,-0.04829470805198899,-0.07810111928752385,-0.020406876903893312 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.03808248493046715,0.07083333333333333,-0.032750848402866184,-0.05311419245427756,-0.013232475823288337 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.03803862528134434,0.08222222222222222,-0.04418359694087788,-0.08141855078897617,-0.010068958967455537 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.05961413544748199,0.10534099765083538,-0.04572686220335339,-0.0945324843133182,0.0013826152737870653 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.11937446972537334,0.19087821976360195,-0.0715037500382286,-0.14653335805251105,0.0024404086896257886 -HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.029807067723740994,0.04305045836052295,-0.013243390636781958,-0.0384699484285053,0.009001188257215744 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.5459065192518,0.048947804264411254,0.020457464895754698,0.07937486042024944 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.032614680056597,0.010077252856861799,0.0031023328080865167,0.020747831469716746 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.2979086632025005,0.1687580034641662,0.07155554235306978,0.2774480293319457 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.47210862793233316,0.14455803873433354,0.039983410998719326,0.24345803450121287 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.16997512612276583,0.18002487387723415,0.07716133175732295,0.28800506073439314 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.12138679992279434,0.13662641968612688,0.06263029181255392,0.21962079097314244 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.0509966130685172,0.055910582846923976,0.02846771961923004,0.08615472760019419 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.07317468265746373,0.06008822497603074,0.03135873418626387,0.09056310898748074 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.04159745451771191,0.05832264603511714,0.025782275188904233,0.09329060029745202 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.03803862528134434,0.04112804138532232,0.020686739564114716,0.06401784999313889 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.03808248493046715,0.032750848402866184,0.015999736614523856,0.050405995011715395 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.03803862528134434,0.048628041385322326,0.0213051106695626,0.0783005935087021 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.05961413544748199,0.040723161483708394,0.01119386649032003,0.07329717629672271 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.11937446972537334,0.05250218823287424,0.007045304398280664,0.0969215226731168 +HLA+peptide LR LOSO,HLA-only LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.029807067723740994,0.02513657764413305,0.0029390636108198695,0.04945687861432687 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5948543235162113,0.6188880671078999,-0.02403374359168864,-0.06339992272149965,0.015856466810683638 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.0426919329134588,0.04488912649824684,-0.0021971935847880444,-0.010770780756085853,0.008134134304698228 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.4666666666666667,0.45,0.016666666666666663,-0.09836065573770492,0.1273167155425218 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.6166666666666667,0.5833333333333334,0.033333333333333326,-0.08932629870129873,0.15517241379310343 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.35,0.2916666666666667,0.05833333333333329,-0.03448275862068961,0.15384615384615383 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.2580132196089212,0.2060379930533435,0.05197522655557771,-0.024532424987950337,0.1282996253126007 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.10690719591544118,0.10329449749236344,0.003612698423077737,-0.025988054059977353,0.035371420451524464 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.13326290763349447,0.13007923122688464,0.0031836764066098255,-0.031152731601539863,0.037255786595359934 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.09992010055282904,0.08624215640883011,0.013677944143998935,-0.01777065475939925,0.045118767142931934 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.07916666666666666,0.08633333333333333,-0.007166666666666668,-0.027499999999999997,0.015695454545454523 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.07083333333333333,0.07083333333333333,0.0,-0.013890027322404365,0.01428571428571429 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.08666666666666667,0.08222222222222222,0.004444444444444445,-0.023503056404556823,0.03114754098360656 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.10033729693119038,0.10534099765083538,-0.005003700719644996,-0.04396245631017302,0.03553267380973812 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.17187665795824758,0.19087821976360195,-0.019001561805354367,-0.08353079239282275,0.042516123849899154 +HLA+peptide LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.05494364536787404,0.04305045836052295,0.011893187007351091,-0.009024639249381839,0.035618348876926255 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,auroc,17475,465,70,0.5459065192518,0.6188880671078999,-0.0729815478560999,-0.12875313741264985,-0.015537675879811655 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,average_precision,17475,465,70,0.032614680056597,0.04488912649824684,-0.012274446441649843,-0.024341311717841006,-0.00250709977379717 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@10,17475,465,70,0.2979086632025005,0.45,-0.15209133679749953,-0.28980508854756826,-0.021391401117801596 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@20,17475,465,70,0.47210862793233316,0.5833333333333334,-0.11122470540100021,-0.2410105261552087,0.021158512901603768 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,hitrate@5,17475,465,70,0.16997512612276583,0.2916666666666667,-0.12169154054390086,-0.23840263822057875,-0.016857306346675458 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,mrr,17475,465,70,0.12138679992279434,0.2060379930533435,-0.08465119313054915,-0.1682251905117228,-0.013927669926753318 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@10,17475,465,70,0.0509966130685172,0.10329449749236344,-0.05229788442384624,-0.09277793584475871,-0.014818739588729708 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@20,17475,465,70,0.07317468265746373,0.13007923122688464,-0.05690454856942091,-0.10314165793053343,-0.013756379356776267 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,ndcg@5,17475,465,70,0.04159745451771191,0.08624215640883011,-0.0446447018911182,-0.08615668756449882,-0.00797460488548041 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@10,17475,465,70,0.03803862528134434,0.08633333333333333,-0.04829470805198899,-0.07810111928752385,-0.020406876903893312 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@20,17475,465,70,0.03808248493046715,0.07083333333333333,-0.032750848402866184,-0.05311419245427756,-0.013232475823288337 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,precision@5,17475,465,70,0.03803862528134434,0.08222222222222222,-0.04418359694087788,-0.08141855078897617,-0.010068958967455537 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@10,17475,465,70,0.05961413544748199,0.10534099765083538,-0.04572686220335339,-0.0945324843133182,0.0013826152737870653 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@20,17475,465,70,0.11937446972537334,0.19087821976360195,-0.0715037500382286,-0.14653335805251105,0.0024404086896257886 +HLA-only LR LOSO,Peptide LR LOSO,immunogenicity_loso,recall@5,17475,465,70,0.029807067723740994,0.04305045836052295,-0.013243390636781958,-0.0384699484285053,0.009001188257215744 diff --git a/results/analysis/improve/exact_peptide_free/missingness.csv b/results/analysis/improve/exact_peptide_free/missingness.csv index 13b5739..8bc2a84 100644 --- a/results/analysis/improve/exact_peptide_free/missingness.csv +++ b/results/analysis/improve/exact_peptide_free/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,17440 -MHCflurry,predicted,17440 -PRIME,predicted,17440 +predictor,status,count +BigMHC,predicted,17440 +MHCflurry,predicted,17440 +PRIME,predicted,17440 diff --git a/results/analysis/improve/exact_peptide_free/paired_differences.csv b/results/analysis/improve/exact_peptide_free/paired_differences.csv index 5a68b15..f4bcd81 100644 --- a/results/analysis/improve/exact_peptide_free/paired_differences.csv +++ b/results/analysis/improve/exact_peptide_free/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,17440,464,70,0.5451536296678475,0.5962768939517047,-0.05112326428385716,-0.09208135581210583,-0.007775698279297643 -BigMHC,PRIME,immunogenicity,average_precision,17440,464,70,0.03168433497806979,0.03950436924868143,-0.007820034270611634,-0.01687301958278144,-0.0007172073562236994 -BigMHC,PRIME,immunogenicity,hitrate@10,17440,464,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 -BigMHC,PRIME,immunogenicity,hitrate@20,17440,464,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 -BigMHC,PRIME,immunogenicity,hitrate@5,17440,464,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 -BigMHC,PRIME,immunogenicity,mrr,17440,464,70,0.18012307209333414,0.212032849396242,-0.031909777302907866,-0.11836977453583092,0.057584532587463225 -BigMHC,PRIME,immunogenicity,ndcg@10,17440,464,70,0.0778599427109938,0.1119592607262639,-0.034099318015270094,-0.073745379964503,0.006438278997035955 -BigMHC,PRIME,immunogenicity,ndcg@20,17440,464,70,0.10337542318970296,0.15115428649810322,-0.04777886330840027,-0.08653657001922378,-0.008577442762943959 -BigMHC,PRIME,immunogenicity,ndcg@5,17440,464,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 -BigMHC,PRIME,immunogenicity,precision@10,17440,464,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 -BigMHC,PRIME,immunogenicity,precision@20,17440,464,70,0.04833333333333333,0.07,-0.021666666666666674,-0.038709677419354833,-0.005930957128614168 -BigMHC,PRIME,immunogenicity,precision@5,17440,464,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 -BigMHC,PRIME,immunogenicity,recall@10,17440,464,70,0.07884220626867686,0.13439881633960582,-0.05555661007092896,-0.11392996669626779,-0.0030021905986799065 -BigMHC,PRIME,immunogenicity,recall@20,17440,464,70,0.14526966526192533,0.25951379903779287,-0.11424413377586753,-0.18672131749168178,-0.0445877321686607 -BigMHC,PRIME,immunogenicity,recall@5,17440,464,70,0.03804563492063492,0.058938290023816335,-0.020892655103181418,-0.04921667171932762,0.009518724498420048 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,17440,464,70,0.5451536296678475,0.5962768939517047,-0.05112326428385716,-0.09208135581210583,-0.007775698279297643 +BigMHC,PRIME,immunogenicity,average_precision,17440,464,70,0.03168433497806979,0.03950436924868143,-0.007820034270611634,-0.01687301958278144,-0.0007172073562236994 +BigMHC,PRIME,immunogenicity,hitrate@10,17440,464,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 +BigMHC,PRIME,immunogenicity,hitrate@20,17440,464,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 +BigMHC,PRIME,immunogenicity,hitrate@5,17440,464,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 +BigMHC,PRIME,immunogenicity,mrr,17440,464,70,0.18012307209333414,0.212032849396242,-0.031909777302907866,-0.11836977453583092,0.057584532587463225 +BigMHC,PRIME,immunogenicity,ndcg@10,17440,464,70,0.0778599427109938,0.1119592607262639,-0.034099318015270094,-0.073745379964503,0.006438278997035955 +BigMHC,PRIME,immunogenicity,ndcg@20,17440,464,70,0.10337542318970296,0.15115428649810322,-0.04777886330840027,-0.08653657001922378,-0.008577442762943959 +BigMHC,PRIME,immunogenicity,ndcg@5,17440,464,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 +BigMHC,PRIME,immunogenicity,precision@10,17440,464,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 +BigMHC,PRIME,immunogenicity,precision@20,17440,464,70,0.04833333333333333,0.07,-0.021666666666666674,-0.038709677419354833,-0.005930957128614168 +BigMHC,PRIME,immunogenicity,precision@5,17440,464,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 +BigMHC,PRIME,immunogenicity,recall@10,17440,464,70,0.07884220626867686,0.13439881633960582,-0.05555661007092896,-0.11392996669626779,-0.0030021905986799065 +BigMHC,PRIME,immunogenicity,recall@20,17440,464,70,0.14526966526192533,0.25951379903779287,-0.11424413377586753,-0.18672131749168178,-0.0445877321686607 +BigMHC,PRIME,immunogenicity,recall@5,17440,464,70,0.03804563492063492,0.058938290023816335,-0.020892655103181418,-0.04921667171932762,0.009518724498420048 diff --git a/results/analysis/improve/fixed/missingness.csv b/results/analysis/improve/fixed/missingness.csv index e94710d..eece4da 100644 --- a/results/analysis/improve/fixed/missingness.csv +++ b/results/analysis/improve/fixed/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -MHCflurry,predicted,17475 -BigMHC,predicted,17475 -PRIME,predicted,17475 +predictor,status,count +MHCflurry,predicted,17475 +BigMHC,predicted,17475 +PRIME,predicted,17475 diff --git a/results/analysis/improve/fixed/paired_differences.csv b/results/analysis/improve/fixed/paired_differences.csv index ea4127e..4c7d40e 100644 --- a/results/analysis/improve/fixed/paired_differences.csv +++ b/results/analysis/improve/fixed/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,17475,465,70,0.5458288293413741,0.5969085863470571,-0.05107975700568301,-0.09216109070862642,-0.007728222469350853 -BigMHC,PRIME,immunogenicity,average_precision,17475,465,70,0.03186133541911459,0.039638725720073424,-0.007777390300958836,-0.01684074393543032,-0.0006004169764923337 -BigMHC,PRIME,immunogenicity,hitrate@10,17475,465,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 -BigMHC,PRIME,immunogenicity,hitrate@20,17475,465,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 -BigMHC,PRIME,immunogenicity,hitrate@5,17475,465,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 -BigMHC,PRIME,immunogenicity,mrr,17475,465,70,0.179799793455899,0.2118375315199005,-0.03203773806400151,-0.11850015573659703,0.05702018960577634 -BigMHC,PRIME,immunogenicity,ndcg@10,17475,465,70,0.07780339513492723,0.1119592607262639,-0.03415586559133667,-0.07374672633536174,0.006436910910518223 -BigMHC,PRIME,immunogenicity,ndcg@20,17475,465,70,0.10393761632591698,0.15170159598198435,-0.047763979656067365,-0.08653882964682626,-0.008579696667541232 -BigMHC,PRIME,immunogenicity,ndcg@5,17475,465,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 -BigMHC,PRIME,immunogenicity,precision@10,17475,465,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 -BigMHC,PRIME,immunogenicity,precision@20,17475,465,70,0.04916666666666667,0.07083333333333333,-0.02166666666666666,-0.038709677419354833,-0.005930957128614168 -BigMHC,PRIME,immunogenicity,precision@5,17475,465,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 -BigMHC,PRIME,immunogenicity,recall@10,17475,465,70,0.07884220626867686,0.13437829088804587,-0.055536084619369014,-0.1139294619720491,-0.002983728978855778 -BigMHC,PRIME,immunogenicity,recall@20,17475,465,70,0.1458033270024836,0.2600474607783511,-0.1142441337758675,-0.18672131749168178,-0.044587732168660715 -BigMHC,PRIME,immunogenicity,recall@5,17475,465,70,0.03804563492063492,0.058917764572256406,-0.02087212965162149,-0.04917484913513216,0.009519246331934279 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,17475,465,70,0.5458288293413741,0.5969085863470571,-0.05107975700568301,-0.09216109070862642,-0.007728222469350853 +BigMHC,PRIME,immunogenicity,average_precision,17475,465,70,0.03186133541911459,0.039638725720073424,-0.007777390300958836,-0.01684074393543032,-0.0006004169764923337 +BigMHC,PRIME,immunogenicity,hitrate@10,17475,465,70,0.38333333333333336,0.48333333333333334,-0.09999999999999998,-0.2419354838709677,0.04918032786885246 +BigMHC,PRIME,immunogenicity,hitrate@20,17475,465,70,0.6,0.7,-0.09999999999999998,-0.2142857142857143,0.01694915254237284 +BigMHC,PRIME,immunogenicity,hitrate@5,17475,465,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 +BigMHC,PRIME,immunogenicity,mrr,17475,465,70,0.179799793455899,0.2118375315199005,-0.03203773806400151,-0.11850015573659703,0.05702018960577634 +BigMHC,PRIME,immunogenicity,ndcg@10,17475,465,70,0.07780339513492723,0.1119592607262639,-0.03415586559133667,-0.07374672633536174,0.006436910910518223 +BigMHC,PRIME,immunogenicity,ndcg@20,17475,465,70,0.10393761632591698,0.15170159598198435,-0.047763979656067365,-0.08653882964682626,-0.008579696667541232 +BigMHC,PRIME,immunogenicity,ndcg@5,17475,465,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 +BigMHC,PRIME,immunogenicity,precision@10,17475,465,70,0.05333333333333334,0.08,-0.026666666666666665,-0.05,-0.005084745762711867 +BigMHC,PRIME,immunogenicity,precision@20,17475,465,70,0.04916666666666667,0.07083333333333333,-0.02166666666666666,-0.038709677419354833,-0.005930957128614168 +BigMHC,PRIME,immunogenicity,precision@5,17475,465,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 +BigMHC,PRIME,immunogenicity,recall@10,17475,465,70,0.07884220626867686,0.13437829088804587,-0.055536084619369014,-0.1139294619720491,-0.002983728978855778 +BigMHC,PRIME,immunogenicity,recall@20,17475,465,70,0.1458033270024836,0.2600474607783511,-0.1142441337758675,-0.18672131749168178,-0.044587732168660715 +BigMHC,PRIME,immunogenicity,recall@5,17475,465,70,0.03804563492063492,0.058917764572256406,-0.02087212965162149,-0.04917484913513216,0.009519246331934279 diff --git a/results/analysis/improve/length_9_10/missingness.csv b/results/analysis/improve/length_9_10/missingness.csv index 9662fff..5febff4 100644 --- a/results/analysis/improve/length_9_10/missingness.csv +++ b/results/analysis/improve/length_9_10/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,15234 -MHCflurry,predicted,15234 -PRIME,predicted,15234 +predictor,status,count +BigMHC,predicted,15234 +MHCflurry,predicted,15234 +PRIME,predicted,15234 diff --git a/results/analysis/improve/length_9_10/paired_differences.csv b/results/analysis/improve/length_9_10/paired_differences.csv index ebcb3e6..e4d4230 100644 --- a/results/analysis/improve/length_9_10/paired_differences.csv +++ b/results/analysis/improve/length_9_10/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,15234,411,70,0.5473864923206571,0.6045464625320058,-0.057159970211348776,-0.09347508009984684,-0.019669288014515205 -BigMHC,PRIME,immunogenicity,average_precision,15234,411,70,0.03273558646002369,0.04101285657052852,-0.00827727011050483,-0.017562909878109576,-0.0007460626001640635 -BigMHC,PRIME,immunogenicity,hitrate@10,15234,411,70,0.4576271186440678,0.4915254237288136,-0.03389830508474578,-0.17543859649122812,0.10719866071428542 -BigMHC,PRIME,immunogenicity,hitrate@20,15234,411,70,0.6440677966101694,0.711864406779661,-0.06779661016949157,-0.190521978021978,0.05263157894736836 -BigMHC,PRIME,immunogenicity,hitrate@5,15234,411,70,0.2542372881355932,0.3559322033898305,-0.1016949152542373,-0.2333333333333333,0.01694915254237289 -BigMHC,PRIME,immunogenicity,mrr,15234,411,70,0.19875514441408423,0.21560258035172208,-0.01684743593763785,-0.09998529680350954,0.06814529199513045 -BigMHC,PRIME,immunogenicity,ndcg@10,15234,411,70,0.08832856137958059,0.11811528728499848,-0.029786725905417893,-0.06966966753810726,0.011317563351223416 -BigMHC,PRIME,immunogenicity,ndcg@20,15234,411,70,0.11776733424221655,0.16343406460510762,-0.04566673036289107,-0.08670885312190944,-0.0058556463640528425 -BigMHC,PRIME,immunogenicity,ndcg@5,15234,411,70,0.06958795373149147,0.10196400897206478,-0.03237605524057331,-0.0772162454749475,0.011905312620938384 -BigMHC,PRIME,immunogenicity,precision@10,15234,411,70,0.0576271186440678,0.08135593220338984,-0.02372881355932204,-0.046269347705914864,-0.0034468147282291067 -BigMHC,PRIME,immunogenicity,precision@20,15234,411,70,0.05338983050847458,0.07203389830508475,-0.018644067796610167,-0.03359442349137931,-0.004310344827586202 -BigMHC,PRIME,immunogenicity,precision@5,15234,411,70,0.054237288135593226,0.10508474576271187,-0.05084745762711865,-0.08888888888888888,-0.016666666666666663 -BigMHC,PRIME,immunogenicity,recall@10,15234,411,70,0.09148901945512115,0.14732567338499541,-0.05583665392987426,-0.11633721127617748,-0.0006664995785759249 -BigMHC,PRIME,immunogenicity,recall@20,15234,411,70,0.166508324414212,0.283096470384606,-0.11658814597039399,-0.19161496564990732,-0.044268565113084475 -BigMHC,PRIME,immunogenicity,recall@5,15234,411,70,0.0489581040428498,0.06835276305615288,-0.019394659013303082,-0.053260956817491156,0.015083855449321512 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,15234,411,70,0.5473864923206571,0.6045464625320058,-0.057159970211348776,-0.09347508009984684,-0.019669288014515205 +BigMHC,PRIME,immunogenicity,average_precision,15234,411,70,0.03273558646002369,0.04101285657052852,-0.00827727011050483,-0.017562909878109576,-0.0007460626001640635 +BigMHC,PRIME,immunogenicity,hitrate@10,15234,411,70,0.4576271186440678,0.4915254237288136,-0.03389830508474578,-0.17543859649122812,0.10719866071428542 +BigMHC,PRIME,immunogenicity,hitrate@20,15234,411,70,0.6440677966101694,0.711864406779661,-0.06779661016949157,-0.190521978021978,0.05263157894736836 +BigMHC,PRIME,immunogenicity,hitrate@5,15234,411,70,0.2542372881355932,0.3559322033898305,-0.1016949152542373,-0.2333333333333333,0.01694915254237289 +BigMHC,PRIME,immunogenicity,mrr,15234,411,70,0.19875514441408423,0.21560258035172208,-0.01684743593763785,-0.09998529680350954,0.06814529199513045 +BigMHC,PRIME,immunogenicity,ndcg@10,15234,411,70,0.08832856137958059,0.11811528728499848,-0.029786725905417893,-0.06966966753810726,0.011317563351223416 +BigMHC,PRIME,immunogenicity,ndcg@20,15234,411,70,0.11776733424221655,0.16343406460510762,-0.04566673036289107,-0.08670885312190944,-0.0058556463640528425 +BigMHC,PRIME,immunogenicity,ndcg@5,15234,411,70,0.06958795373149147,0.10196400897206478,-0.03237605524057331,-0.0772162454749475,0.011905312620938384 +BigMHC,PRIME,immunogenicity,precision@10,15234,411,70,0.0576271186440678,0.08135593220338984,-0.02372881355932204,-0.046269347705914864,-0.0034468147282291067 +BigMHC,PRIME,immunogenicity,precision@20,15234,411,70,0.05338983050847458,0.07203389830508475,-0.018644067796610167,-0.03359442349137931,-0.004310344827586202 +BigMHC,PRIME,immunogenicity,precision@5,15234,411,70,0.054237288135593226,0.10508474576271187,-0.05084745762711865,-0.08888888888888888,-0.016666666666666663 +BigMHC,PRIME,immunogenicity,recall@10,15234,411,70,0.09148901945512115,0.14732567338499541,-0.05583665392987426,-0.11633721127617748,-0.0006664995785759249 +BigMHC,PRIME,immunogenicity,recall@20,15234,411,70,0.166508324414212,0.283096470384606,-0.11658814597039399,-0.19161496564990732,-0.044268565113084475 +BigMHC,PRIME,immunogenicity,recall@5,15234,411,70,0.0489581040428498,0.06835276305615288,-0.019394659013303082,-0.053260956817491156,0.015083855449321512 diff --git a/results/analysis/improve/near_overlap_free/missingness.csv b/results/analysis/improve/near_overlap_free/missingness.csv index 4a6845d..7b55edc 100644 --- a/results/analysis/improve/near_overlap_free/missingness.csv +++ b/results/analysis/improve/near_overlap_free/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,17457 -MHCflurry,predicted,17457 -PRIME,predicted,17457 +predictor,status,count +BigMHC,predicted,17457 +MHCflurry,predicted,17457 +PRIME,predicted,17457 diff --git a/results/analysis/improve/near_overlap_free/paired_differences.csv b/results/analysis/improve/near_overlap_free/paired_differences.csv index a03a4cf..7d39ed5 100644 --- a/results/analysis/improve/near_overlap_free/paired_differences.csv +++ b/results/analysis/improve/near_overlap_free/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,17457,463,70,0.5463723570585578,0.5959065466124367,-0.04953418955387889,-0.09028394644135934,-0.005571772811015889 -BigMHC,PRIME,immunogenicity,average_precision,17457,463,70,0.03180240566708497,0.03944557860693173,-0.00764317293984676,-0.016691321197734324,-0.0004495652976370504 -BigMHC,PRIME,immunogenicity,hitrate@10,17457,463,70,0.4,0.4666666666666667,-0.06666666666666665,-0.2068965517241379,0.08200136612021842 -BigMHC,PRIME,immunogenicity,hitrate@20,17457,463,70,0.5833333333333334,0.6833333333333333,-0.09999999999999998,-0.2281061259706644,0.01950757575757418 -BigMHC,PRIME,immunogenicity,hitrate@5,17457,463,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 -BigMHC,PRIME,immunogenicity,mrr,17457,463,70,0.17925826810475948,0.20999589027457913,-0.03073762216981965,-0.11675118827663264,0.058761820430309405 -BigMHC,PRIME,immunogenicity,ndcg@10,17457,463,70,0.08262114224022538,0.10875190364903926,-0.026130761408813888,-0.06586530107897698,0.016543074144428887 -BigMHC,PRIME,immunogenicity,ndcg@20,17457,463,70,0.10475573278675282,0.1484942389047597,-0.04373850611800689,-0.08338951426895236,-0.0032750982543430823 -BigMHC,PRIME,immunogenicity,ndcg@5,17457,463,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 -BigMHC,PRIME,immunogenicity,precision@10,17457,463,70,0.055,0.07833333333333334,-0.023333333333333338,-0.046434523809523814,-0.0016393442622950893 -BigMHC,PRIME,immunogenicity,precision@20,17457,463,70,0.04833333333333333,0.07,-0.021666666666666674,-0.03834482758620691,-0.0060319257744009475 -BigMHC,PRIME,immunogenicity,precision@5,17457,463,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 -BigMHC,PRIME,immunogenicity,recall@10,17457,463,70,0.09550887293534352,0.12644178295153793,-0.030932910016194415,-0.09358394741485443,0.03146490109353547 -BigMHC,PRIME,immunogenicity,recall@20,17457,463,70,0.15175570795486457,0.2521109528418432,-0.10035524488697864,-0.1788990282645159,-0.01936635838545947 -BigMHC,PRIME,immunogenicity,recall@5,17457,463,70,0.03804563492063492,0.0593145899690818,-0.021268955048446885,-0.049942376972316196,0.00936711535770567 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,17457,463,70,0.5463723570585578,0.5959065466124367,-0.04953418955387889,-0.09028394644135934,-0.005571772811015889 +BigMHC,PRIME,immunogenicity,average_precision,17457,463,70,0.03180240566708497,0.03944557860693173,-0.00764317293984676,-0.016691321197734324,-0.0004495652976370504 +BigMHC,PRIME,immunogenicity,hitrate@10,17457,463,70,0.4,0.4666666666666667,-0.06666666666666665,-0.2068965517241379,0.08200136612021842 +BigMHC,PRIME,immunogenicity,hitrate@20,17457,463,70,0.5833333333333334,0.6833333333333333,-0.09999999999999998,-0.2281061259706644,0.01950757575757418 +BigMHC,PRIME,immunogenicity,hitrate@5,17457,463,70,0.21666666666666667,0.35,-0.1333333333333333,-0.2712249614791988,0.0 +BigMHC,PRIME,immunogenicity,mrr,17457,463,70,0.17925826810475948,0.20999589027457913,-0.03073762216981965,-0.11675118827663264,0.058761820430309405 +BigMHC,PRIME,immunogenicity,ndcg@10,17457,463,70,0.08262114224022538,0.10875190364903926,-0.026130761408813888,-0.06586530107897698,0.016543074144428887 +BigMHC,PRIME,immunogenicity,ndcg@20,17457,463,70,0.10475573278675282,0.1484942389047597,-0.04373850611800689,-0.08338951426895236,-0.0032750982543430823 +BigMHC,PRIME,immunogenicity,ndcg@5,17457,463,70,0.05999687767867254,0.09908070239863123,-0.03908382471995869,-0.08595963282450478,0.00877791891781991 +BigMHC,PRIME,immunogenicity,precision@10,17457,463,70,0.055,0.07833333333333334,-0.023333333333333338,-0.046434523809523814,-0.0016393442622950893 +BigMHC,PRIME,immunogenicity,precision@20,17457,463,70,0.04833333333333333,0.07,-0.021666666666666674,-0.03834482758620691,-0.0060319257744009475 +BigMHC,PRIME,immunogenicity,precision@5,17457,463,70,0.04666666666666667,0.10333333333333333,-0.056666666666666664,-0.09526455026455029,-0.019991803278688575 +BigMHC,PRIME,immunogenicity,recall@10,17457,463,70,0.09550887293534352,0.12644178295153793,-0.030932910016194415,-0.09358394741485443,0.03146490109353547 +BigMHC,PRIME,immunogenicity,recall@20,17457,463,70,0.15175570795486457,0.2521109528418432,-0.10035524488697864,-0.1788990282645159,-0.01936635838545947 +BigMHC,PRIME,immunogenicity,recall@5,17457,463,70,0.03804563492063492,0.0593145899690818,-0.021268955048446885,-0.049942376972316196,0.00936711535770567 diff --git a/results/analysis/improve/peptide_sensitivity/missingness.csv b/results/analysis/improve/peptide_sensitivity/missingness.csv index 0c7d4f3..31e1cf8 100644 --- a/results/analysis/improve/peptide_sensitivity/missingness.csv +++ b/results/analysis/improve/peptide_sensitivity/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,15508 -MHCflurry,predicted,15508 -PRIME,predicted,15508 +predictor,status,count +BigMHC,predicted,15508 +MHCflurry,predicted,15508 +PRIME,predicted,15508 diff --git a/results/analysis/improve/peptide_sensitivity/paired_differences.csv b/results/analysis/improve/peptide_sensitivity/paired_differences.csv index ea4cb32..834b1e5 100644 --- a/results/analysis/improve/peptide_sensitivity/paired_differences.csv +++ b/results/analysis/improve/peptide_sensitivity/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5570005249823103,0.6197087694285887,-0.06270824444627832,-0.10328418049333872,-0.021200824425258242 -BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.035725091615924305,0.047085792465602884,-0.011360700849678579,-0.021756193831568165,-0.002717924156678858 -BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5333333333333333,-0.08333333333333331,-0.21666666666666667,0.05084745762711862 -BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6333333333333333,0.7666666666666667,-0.13333333333333341,-0.24137931034482762,-0.030500000000001474 -BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.25,0.38333333333333336,-0.13333333333333336,-0.27118644067796605,0.0 -BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.202555293905601,0.23299789338847923,-0.030442599482878224,-0.11356606790906844,0.058125805978220195 -BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.08917621083183522,0.13067786000916146,-0.04150164917732624,-0.08349233414665562,0.0017312696400933877 -BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.12093419366128383,0.1777591237628782,-0.05682493010159437,-0.09535555606055685,-0.016967811593794378 -BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.06847588281051294,0.11266929096539889,-0.044193408154885955,-0.09153757425491718,0.004235181157039931 -BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.06166666666666667,0.09166666666666667,-0.030000000000000006,-0.056365835777126096,-0.006666666666666668 -BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.060000000000000005,0.08416666666666667,-0.024166666666666663,-0.04051724137931034,-0.00872742200328409 -BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.05333333333333334,0.11666666666666667,-0.06333333333333332,-0.10345135467980297,-0.02413793103448276 -BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.08778671083818143,0.16338126475170087,-0.07559455391351944,-0.14399416476069116,-0.012787164224664251 -BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.16564418302755016,0.30375703286834116,-0.138112849840791,-0.21430548970982094,-0.06118383147473153 -BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.04356150793650793,0.07171790496334107,-0.02815639702683314,-0.06006275148479949,0.005094435672090141 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5570005249823103,0.6197087694285887,-0.06270824444627832,-0.10328418049333872,-0.021200824425258242 +BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.035725091615924305,0.047085792465602884,-0.011360700849678579,-0.021756193831568165,-0.002717924156678858 +BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5333333333333333,-0.08333333333333331,-0.21666666666666667,0.05084745762711862 +BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6333333333333333,0.7666666666666667,-0.13333333333333341,-0.24137931034482762,-0.030500000000001474 +BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.25,0.38333333333333336,-0.13333333333333336,-0.27118644067796605,0.0 +BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.202555293905601,0.23299789338847923,-0.030442599482878224,-0.11356606790906844,0.058125805978220195 +BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.08917621083183522,0.13067786000916146,-0.04150164917732624,-0.08349233414665562,0.0017312696400933877 +BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.12093419366128383,0.1777591237628782,-0.05682493010159437,-0.09535555606055685,-0.016967811593794378 +BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.06847588281051294,0.11266929096539889,-0.044193408154885955,-0.09153757425491718,0.004235181157039931 +BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.06166666666666667,0.09166666666666667,-0.030000000000000006,-0.056365835777126096,-0.006666666666666668 +BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.060000000000000005,0.08416666666666667,-0.024166666666666663,-0.04051724137931034,-0.00872742200328409 +BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.05333333333333334,0.11666666666666667,-0.06333333333333332,-0.10345135467980297,-0.02413793103448276 +BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.08778671083818143,0.16338126475170087,-0.07559455391351944,-0.14399416476069116,-0.012787164224664251 +BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.16564418302755016,0.30375703286834116,-0.138112849840791,-0.21430548970982094,-0.06118383147473153 +BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.04356150793650793,0.07171790496334107,-0.02815639702683314,-0.06006275148479949,0.005094435672090141 diff --git a/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv b/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv index 0c7d4f3..31e1cf8 100644 --- a/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv +++ b/results/analysis/improve/peptide_sensitivity_hla_rank/missingness.csv @@ -1,4 +1,4 @@ -predictor,status,count -BigMHC,predicted,15508 -MHCflurry,predicted,15508 -PRIME,predicted,15508 +predictor,status,count +BigMHC,predicted,15508 +MHCflurry,predicted,15508 +PRIME,predicted,15508 diff --git a/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv b/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv index c7c16fa..f4a4baa 100644 --- a/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv +++ b/results/analysis/improve/peptide_sensitivity_hla_rank/paired_differences.csv @@ -1,16 +1,16 @@ -left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high -BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5900546578572645,0.6237143200776764,-0.033659662220411835,-0.06399324796039386,-0.004217013456002351 -BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.04431785814256973,0.0480246940858717,-0.00370683594330197,-0.0119833307880007,0.006291990015651395 -BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5666666666666667,-0.11666666666666664,-0.2459016393442623,0.014936854190585514 -BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6666666666666666,0.6666666666666666,0.0,-0.09375,0.09523809523809523 -BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.31666666666666665,0.38333333333333336,-0.06666666666666671,-0.1896756978653531,0.051724137931034475 -BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.23930852333307953,0.2560274317551143,-0.016718908422034745,-0.10748366597900222,0.07806786807751256 -BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.10539836768508622,0.1293553240052982,-0.023956956320211983,-0.06550454575548117,0.020390104084166288 -BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.14260274243018445,0.16198440048673515,-0.019381658056550705,-0.05645245831761408,0.02085912104839368 -BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.0962671022951806,0.11131066569892652,-0.015043563403745921,-0.061799664785384734,0.03481131928464634 -BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.07166666666666667,0.09,-0.018333333333333326,-0.03870967741935484,0.0017241379310344862 -BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.06583333333333334,0.07833333333333334,-0.012499999999999997,-0.025806451612903222,0.0 -BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.08,0.10666666666666667,-0.026666666666666672,-0.0655750512295082,0.012130681818181761 -BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.092826266964198,0.15148871285914897,-0.05866244589495097,-0.11852260865010635,-0.0013937041464451819 -BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.20158400709237423,0.2401696615870044,-0.03858565449463017,-0.08995684333749829,0.00712539702249144 -BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.05976333452626556,0.06611274623318233,-0.006349411706916774,-0.04051912971569411,0.027851384442322387 +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,PRIME,immunogenicity,auroc,15508,454,70,0.5900546578572645,0.6237143200776764,-0.033659662220411835,-0.06399324796039386,-0.004217013456002351 +BigMHC,PRIME,immunogenicity,average_precision,15508,454,70,0.04431785814256973,0.0480246940858717,-0.00370683594330197,-0.0119833307880007,0.006291990015651395 +BigMHC,PRIME,immunogenicity,hitrate@10,15508,454,70,0.45,0.5666666666666667,-0.11666666666666664,-0.2459016393442623,0.014936854190585514 +BigMHC,PRIME,immunogenicity,hitrate@20,15508,454,70,0.6666666666666666,0.6666666666666666,0.0,-0.09375,0.09523809523809523 +BigMHC,PRIME,immunogenicity,hitrate@5,15508,454,70,0.31666666666666665,0.38333333333333336,-0.06666666666666671,-0.1896756978653531,0.051724137931034475 +BigMHC,PRIME,immunogenicity,mrr,15508,454,70,0.23930852333307953,0.2560274317551143,-0.016718908422034745,-0.10748366597900222,0.07806786807751256 +BigMHC,PRIME,immunogenicity,ndcg@10,15508,454,70,0.10539836768508622,0.1293553240052982,-0.023956956320211983,-0.06550454575548117,0.020390104084166288 +BigMHC,PRIME,immunogenicity,ndcg@20,15508,454,70,0.14260274243018445,0.16198440048673515,-0.019381658056550705,-0.05645245831761408,0.02085912104839368 +BigMHC,PRIME,immunogenicity,ndcg@5,15508,454,70,0.0962671022951806,0.11131066569892652,-0.015043563403745921,-0.061799664785384734,0.03481131928464634 +BigMHC,PRIME,immunogenicity,precision@10,15508,454,70,0.07166666666666667,0.09,-0.018333333333333326,-0.03870967741935484,0.0017241379310344862 +BigMHC,PRIME,immunogenicity,precision@20,15508,454,70,0.06583333333333334,0.07833333333333334,-0.012499999999999997,-0.025806451612903222,0.0 +BigMHC,PRIME,immunogenicity,precision@5,15508,454,70,0.08,0.10666666666666667,-0.026666666666666672,-0.0655750512295082,0.012130681818181761 +BigMHC,PRIME,immunogenicity,recall@10,15508,454,70,0.092826266964198,0.15148871285914897,-0.05866244589495097,-0.11852260865010635,-0.0013937041464451819 +BigMHC,PRIME,immunogenicity,recall@20,15508,454,70,0.20158400709237423,0.2401696615870044,-0.03858565449463017,-0.08995684333749829,0.00712539702249144 +BigMHC,PRIME,immunogenicity,recall@5,15508,454,70,0.05976333452626556,0.06611274623318233,-0.006349411706916774,-0.04051912971569411,0.027851384442322387 diff --git a/results/analysis/rcc/metrics.json b/results/analysis/rcc/metrics.json new file mode 100644 index 0000000..dcb279c --- /dev/null +++ b/results/analysis/rcc/metrics.json @@ -0,0 +1,2911 @@ +{ + "benchmark": "data/processed/rcc_vaccine_benchmark.csv", + "common_support": [ + { + "left": "BigMHC", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "task": "immunogenicity" + }, + { + "left": "BigMHC", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "task": "immunogenicity" + }, + { + "left": "BigMHC", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "task": "immunogenicity" + }, + { + "left": "DeepHLApan", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "task": "immunogenicity" + }, + { + "left": "DeepHLApan", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "task": "immunogenicity" + }, + { + "left": "DeepImmuno-CNN", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "task": "immunogenicity" + } + ], + "config": { + "bootstrap": 2000, + "ks": [ + 5, + 10, + 20 + ], + "seed": 20260820 + }, + "metrics": { + "BigMHC": { + "hla": { + "HLA-A*01:01": { + "auroc": 0.5555555555555556, + "average_precision": 0.9467813051146384, + "brier": 0.7766608713597656, + "n": 10, + "positives": 9 + }, + "HLA-A*02:01": { + "auroc": 0.8055555555555556, + "average_precision": 0.5888888888888888, + "brier": 0.17650530414755602, + "n": 9, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.6666666666666666, + "average_precision": 0.75, + "brier": 0.2554600718003685, + "n": 5, + "positives": 2 + }, + "HLA-A*25:01": { + "auroc": null, + "average_precision": null, + "brier": 5.337579490387601e-05, + "n": 1, + "positives": 0 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.009819210166266542, + "n": 3, + "positives": 0 + }, + "HLA-A*31:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.4452767978083257, + "n": 2, + "positives": 1 + }, + "HLA-A*32:01": { + "auroc": 0.0, + "average_precision": 0.5833333333333333, + "brier": 0.6360571360656988, + "n": 3, + "positives": 2 + }, + "HLA-A*66:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.4588287991777727, + "n": 4, + "positives": 2 + }, + "HLA-B*07:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.27139674555417276, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.5333333333333333, + "average_precision": 0.7250000000000001, + "brier": 0.32692333515866856, + "n": 8, + "positives": 5 + }, + "HLA-B*13:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.13057078448859577, + "n": 2, + "positives": 1 + }, + "HLA-B*18:01": { + "auroc": 0.25, + "average_precision": 0.5833333333333333, + "brier": 0.5087623762373769, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.5, + "average_precision": 0.5381944444444444, + "brier": 0.2843966461375668, + "n": 10, + "positives": 4 + }, + "HLA-B*27:06": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9825847383470074, + "n": 1, + "positives": 1 + }, + "HLA-B*35:01": { + "auroc": null, + "average_precision": null, + "brier": 0.039145639455604904, + "n": 1, + "positives": 0 + }, + "HLA-B*35:03": { + "auroc": 0.0, + "average_precision": 0.3333333333333333, + "brier": 0.3219427027809941, + "n": 3, + "positives": 1 + }, + "HLA-B*38:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.39194105610017, + "n": 2, + "positives": 1 + }, + "HLA-B*40:01": { + "auroc": 0.4666666666666667, + "average_precision": 0.7226190476190477, + "brier": 0.4048368493871514, + "n": 8, + "positives": 5 + }, + "HLA-B*44:02": { + "auroc": 0.5125, + "average_precision": 0.6842147435897434, + "brier": 0.5705962359098319, + "n": 13, + "positives": 8 + }, + "HLA-B*55:01": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9723301576064242, + "n": 3, + "positives": 3 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.25555068008958337, + "n": 3, + "positives": 2 + }, + "HLA-C*03:03": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.50740073893866, + "n": 3, + "positives": 2 + }, + "HLA-C*05:01": { + "auroc": 0.0, + "average_precision": 0.6388888888888888, + "brier": 0.6178133595147751, + "n": 4, + "positives": 3 + }, + "HLA-C*06:02": { + "auroc": 0.5, + "average_precision": 0.8875, + "brier": 0.6729075510492147, + "n": 5, + "positives": 4 + }, + "HLA-C*07:01": { + "auroc": 0.32857142857142857, + "average_precision": 0.5365388579674294, + "brier": 0.4838775810121097, + "n": 12, + "positives": 7 + }, + "HLA-C*07:02": { + "auroc": 0.3333333333333333, + "average_precision": 0.8055555555555556, + "brier": 0.38905007015972415, + "n": 4, + "positives": 3 + }, + "HLA-C*12:03": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9803353164996088, + "n": 2, + "positives": 2 + } + }, + "metadata": { + "score_direction": "higher", + "source": "results/raw_predictions/rcc/bigmhc-v1.0.csv", + "task": "immunogenicity", + "version": "v1.0" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.6518518518518518, + "ndcg@10": 0.6312873089172458, + "ndcg@20": 0.7738604771130371, + "ndcg@5": 0.5325765609825637, + "positive_bearing_patients": 9, + "precision@10": 0.5611111111111111, + "precision@20": 0.5646005983467284, + "precision@5": 0.5333333333333333, + "recall@10": 0.7159050825717492, + "recall@20": 1.0, + "recall@5": 0.35287798621131955 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.5535088711420252, + "low": 0.39938211912672045 + }, + "average_precision": { + "high": 0.673066296054345, + "low": 0.49899434265684595 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 1.0 + }, + "mrr": { + "high": 0.837037037037037, + "low": 0.45365740740740734 + }, + "ndcg@10": { + "high": 0.7275479609299087, + "low": 0.5328394305376711 + }, + "ndcg@20": { + "high": 0.8561682035271525, + "low": 0.6647997182152893 + }, + "ndcg@5": { + "high": 0.6403384470711495, + "low": 0.4175916065340972 + }, + "precision@10": { + "high": 0.6666666666666666, + "low": 0.45 + }, + "precision@20": { + "high": 0.6469078556462458, + "low": 0.4669648367055487 + }, + "precision@5": { + "high": 0.6444444444444445, + "low": 0.4222222222222222 + }, + "recall@10": { + "high": 0.8320025653358986, + "low": 0.6028407086740422 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.4365211640211639, + "low": 0.28138327721661055 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8762995389962467, + "ndcg@20": 0.8762995389962467, + "ndcg@5": 0.7590911424516827, + "precision@10": 0.7, + "precision@20": 0.6363636363636364, + "precision@5": 0.8, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.5714285714285714 + }, + "RCC-102": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.2, + "ndcg@10": 0.430624116386567, + "ndcg@20": 0.430624116386567, + "ndcg@5": 0.23719771276929622, + "precision@10": 0.25, + "precision@20": 0.25, + "precision@5": 0.2, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.5 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.6395647163312485, + "ndcg@20": 0.8837407779997417, + "ndcg@5": 0.7227265726449519, + "precision@10": 0.5, + "precision@20": 0.5294117647058824, + "precision@5": 0.6, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.41666666666666663, + "ndcg@10": 0.5918332819916706, + "ndcg@20": 0.7898052753909603, + "ndcg@5": 0.46905661104071605, + "precision@10": 0.7, + "precision@20": 0.7368421052631579, + "precision@5": 0.6, + "recall@10": 0.5, + "recall@20": 1.0, + "recall@5": 0.21428571428571427 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.47534276898500794, + "ndcg@20": 0.7203074868223487, + "ndcg@5": 0.36005461457723403, + "precision@10": 0.5, + "precision@20": 0.5625, + "precision@5": 0.4, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.2222222222222222 + }, + "RCC-107": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7729548681688855, + "ndcg@20": 0.8573636767513856, + "ndcg@5": 0.5531464700081437, + "precision@10": 0.5, + "precision@20": 0.5454545454545454, + "precision@5": 0.4, + "recall@10": 0.8333333333333334, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.41666666666666663, + "ndcg@10": 0.46289840701360524, + "ndcg@20": 0.6780957041773528, + "ndcg@5": 0.46905661104071605, + "precision@10": 0.4, + "precision@20": 0.4666666666666667, + "precision@5": 0.6, + "recall@10": 0.5714285714285714, + "recall@20": 1.0, + "recall@5": 0.42857142857142855 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6832491119818491, + "ndcg@20": 0.807381612705331, + "ndcg@5": 0.5118886180624318, + "precision@10": 0.8, + "precision@20": 0.6875, + "precision@5": 0.6, + "recall@10": 0.7272727272727273, + "recall@20": 1.0, + "recall@5": 0.2727272727272727 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7488189704001319, + "ndcg@20": 0.9211261047873995, + "ndcg@5": 0.710970696247901, + "precision@10": 0.7, + "precision@20": 0.6666666666666666, + "precision@5": 0.6, + "recall@10": 0.7, + "recall@20": 1.0, + "recall@5": 0.3 + } + }, + "pooled": { + "auroc": 0.4759748427672956, + "average_precision": 0.5856631462955934, + "brier": 0.44860391491063, + "n": 128, + "positives": 75 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.4759748427672956, + "average_precision": 0.5856631462955934, + "brier": 0.44860391491063, + "n": 128, + "positives": 75 + } + } + }, + "DeepHLApan": { + "hla": { + "HLA-A*01:01": { + "auroc": 0.5555555555555556, + "average_precision": 0.9467813051146384, + "brier": 0.6350796566107824, + "n": 10, + "positives": 9 + }, + "HLA-A*02:01": { + "auroc": 0.5277777777777778, + "average_precision": 0.425, + "brier": 0.30187009644184803, + "n": 9, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.8333333333333334, + "average_precision": 0.8333333333333333, + "brier": 0.21866561025884895, + "n": 5, + "positives": 2 + }, + "HLA-A*25:01": { + "auroc": null, + "average_precision": null, + "brier": 0.5919859914807039, + "n": 1, + "positives": 0 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.6518509783239685, + "n": 3, + "positives": 0 + }, + "HLA-A*31:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.22453314818114337, + "n": 2, + "positives": 1 + }, + "HLA-A*32:01": { + "auroc": 0.5, + "average_precision": 0.8333333333333333, + "brier": 0.29385490350386284, + "n": 3, + "positives": 2 + }, + "HLA-A*66:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.2637184827870923, + "n": 4, + "positives": 2 + }, + "HLA-B*07:02": { + "auroc": 0.75, + "average_precision": 0.8333333333333333, + "brier": 0.24624852418541207, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.4666666666666667, + "average_precision": 0.6983333333333334, + "brier": 0.3105990778488774, + "n": 8, + "positives": 5 + }, + "HLA-B*13:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.409583986666841, + "n": 2, + "positives": 1 + }, + "HLA-B*18:01": { + "auroc": 0.25, + "average_precision": 0.5833333333333333, + "brier": 0.30231696703191596, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.6666666666666666, + "average_precision": 0.6011904761904762, + "brier": 0.36036220595666985, + "n": 10, + "positives": 4 + }, + "HLA-B*27:06": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.18737445074360148, + "n": 1, + "positives": 1 + }, + "HLA-B*35:01": { + "auroc": null, + "average_precision": null, + "brier": 0.5986074455543146, + "n": 1, + "positives": 0 + }, + "HLA-B*35:03": { + "auroc": 0.0, + "average_precision": 0.3333333333333333, + "brier": 0.647851957956463, + "n": 3, + "positives": 1 + }, + "HLA-B*38:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.47431751632294605, + "n": 2, + "positives": 1 + }, + "HLA-B*40:01": { + "auroc": 0.2, + "average_precision": 0.5392857142857144, + "brier": 0.503263362761755, + "n": 8, + "positives": 5 + }, + "HLA-B*44:02": { + "auroc": 0.7625, + "average_precision": 0.8607954545454546, + "brier": 0.29576446274958446, + "n": 13, + "positives": 8 + }, + "HLA-B*55:01": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9321453855555113, + "n": 3, + "positives": 3 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.30637106710129497, + "n": 3, + "positives": 2 + }, + "HLA-C*03:03": { + "auroc": 0.0, + "average_precision": 0.5833333333333333, + "brier": 0.3276111573648398, + "n": 3, + "positives": 2 + }, + "HLA-C*05:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.22813751061878165, + "n": 4, + "positives": 3 + }, + "HLA-C*06:02": { + "auroc": 0.5, + "average_precision": 0.8875, + "brier": 0.19676999380006047, + "n": 5, + "positives": 4 + }, + "HLA-C*07:01": { + "auroc": 0.4714285714285714, + "average_precision": 0.6553030303030304, + "brier": 0.2791138986130422, + "n": 12, + "positives": 7 + }, + "HLA-C*07:02": { + "auroc": 0.6666666666666666, + "average_precision": 0.9166666666666666, + "brier": 0.23313610190493347, + "n": 4, + "positives": 3 + }, + "HLA-C*12:03": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.0001787875426514692, + "n": 2, + "positives": 2 + } + }, + "metadata": { + "score_direction": "higher", + "source": "results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "task": "immunogenicity", + "version": "1.1.1@ac1f4beb" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 0.8888888888888888, + "mrr": 0.7222222222222222, + "ndcg@10": 0.6703258245072994, + "ndcg@20": 0.8085230459234908, + "ndcg@5": 0.6140248752438171, + "positive_bearing_patients": 9, + "precision@10": 0.5611111111111111, + "precision@20": 0.5646005983467284, + "precision@5": 0.6222222222222222, + "recall@10": 0.7264870931537598, + "recall@20": 1.0, + "recall@5": 0.35645342312008976 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.595005662825475, + "low": 0.40889232966390093 + }, + "average_precision": { + "high": 0.7418858290949979, + "low": 0.5420228402887045 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 0.6666666666666666 + }, + "mrr": { + "high": 0.9074074074074074, + "low": 0.5185185185185185 + }, + "ndcg@10": { + "high": 0.773125749993384, + "low": 0.5659855649509037 + }, + "ndcg@20": { + "high": 0.8947689844911606, + "low": 0.6899714059669764 + }, + "ndcg@5": { + "high": 0.7654367605421801, + "low": 0.4067505026362102 + }, + "precision@10": { + "high": 0.6501388888888882, + "low": 0.4611111111111111 + }, + "precision@20": { + "high": 0.6469078556462458, + "low": 0.4669648367055487 + }, + "precision@5": { + "high": 0.7555555555555556, + "low": 0.4222222222222222 + }, + "recall@10": { + "high": 0.8537918871252205, + "low": 0.601262225428892 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.4670554753888087, + "low": 0.23950296616963282 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.7730836500855316, + "ndcg@20": 0.8497584640784245, + "ndcg@5": 0.6130227926089763, + "precision@10": 0.6, + "precision@20": 0.6363636363636364, + "precision@5": 0.6, + "recall@10": 0.8571428571428571, + "recall@20": 1.0, + "recall@5": 0.42857142857142855 + }, + "RCC-102": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 0.0, + "mrr": 0.16666666666666666, + "ndcg@10": 0.4227898344066503, + "ndcg@20": 0.4227898344066503, + "ndcg@5": 0.0, + "precision@10": 0.25, + "precision@20": 0.25, + "precision@5": 0.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.0 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.659224332815107, + "ndcg@20": 0.893591198482684, + "ndcg@5": 0.830419897363192, + "precision@10": 0.5, + "precision@20": 0.5294117647058824, + "precision@5": 0.8, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.4444444444444444 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5683416927717359, + "ndcg@20": 0.8218204494927667, + "ndcg@5": 0.6608397947263839, + "precision@10": 0.6, + "precision@20": 0.7368421052631579, + "precision@5": 0.8, + "recall@10": 0.42857142857142855, + "recall@20": 1.0, + "recall@5": 0.2857142857142857 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5161528906822677, + "ndcg@20": 0.754314697957365, + "ndcg@5": 0.5296347172140421, + "precision@10": 0.5, + "precision@20": 0.5625, + "precision@5": 0.6, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-107": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.944819226755994, + "ndcg@20": 0.944819226755994, + "ndcg@5": 0.8539316501572937, + "precision@10": 0.6, + "precision@20": 0.5454545454545454, + "precision@5": 0.8, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.6666666666666666 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5798612725328837, + "ndcg@20": 0.7275358570132511, + "ndcg@5": 0.38356636737133565, + "precision@10": 0.5, + "precision@20": 0.4666666666666667, + "precision@5": 0.4, + "recall@10": 0.7142857142857143, + "recall@20": 1.0, + "recall@5": 0.2857142857142857 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7914544336296033, + "ndcg@20": 0.9106496109753879, + "ndcg@5": 0.7860137352654724, + "precision@10": 0.8, + "precision@20": 0.6875, + "precision@5": 0.8, + "recall@10": 0.7272727272727273, + "recall@20": 1.0, + "recall@5": 0.36363636363636365 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7772050868859213, + "ndcg@20": 0.9514280741488932, + "ndcg@5": 0.8687949224876582, + "precision@10": 0.7, + "precision@20": 0.6666666666666666, + "precision@5": 0.8, + "recall@10": 0.7, + "recall@20": 1.0, + "recall@5": 0.4 + } + }, + "pooled": { + "auroc": 0.5046540880503144, + "average_precision": 0.6532494332203217, + "brier": 0.361873408840907, + "n": 128, + "positives": 75 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.5046540880503144, + "average_precision": 0.6532494332203217, + "brier": 0.361873408840907, + "n": 128, + "positives": 75 + } + } + }, + "DeepImmuno-CNN": { + "hla": { + "HLA-A*01:01": { + "auroc": null, + "average_precision": 0.9999999999999999, + "brier": 0.3949711436600252, + "n": 6, + "positives": 6 + }, + "HLA-A*02:01": { + "auroc": 0.2777777777777778, + "average_precision": 0.4666666666666667, + "brier": 0.36143655748097425, + "n": 6, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.4841049884591555, + "n": 2, + "positives": 1 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.8053670747968357, + "n": 3, + "positives": 0 + }, + "HLA-B*07:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.11387628173357213, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.75, + "average_precision": 0.6666666666666666, + "brier": 0.3782338557424658, + "n": 4, + "positives": 2 + }, + "HLA-B*18:01": { + "auroc": 0.75, + "average_precision": 0.8333333333333333, + "brier": 0.2261920761561721, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.625, + "average_precision": 0.6499999999999999, + "brier": 0.3437516617313712, + "n": 8, + "positives": 4 + }, + "HLA-B*40:01": { + "auroc": 0.0, + "average_precision": 0.6388888888888888, + "brier": 0.23982545942086936, + "n": 4, + "positives": 3 + }, + "HLA-B*44:02": { + "auroc": 0.6666666666666666, + "average_precision": 0.3333333333333333, + "brier": 0.21491416359774884, + "n": 4, + "positives": 1 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.2952957789199111, + "n": 3, + "positives": 2 + }, + "HLA-C*06:02": { + "auroc": 0.3333333333333333, + "average_precision": 0.8055555555555556, + "brier": 0.25271668358889426, + "n": 4, + "positives": 3 + } + }, + "metadata": { + "score_direction": "higher", + "source": "results/raw_predictions/rcc/deepimmuno-cnn.csv", + "task": "immunogenicity", + "version": "1.0@df42ac5b" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8095238095238095, + "ndcg@10": 0.8191558587095117, + "ndcg@20": 0.8368671618581434, + "ndcg@5": 0.7107525131913099, + "positive_bearing_patients": 7, + "precision@10": 0.616156462585034, + "precision@20": 0.6232993197278911, + "precision@5": 0.6523809523809524, + "recall@10": 0.9642857142857143, + "recall@20": 1.0, + "recall@5": 0.7547619047619047 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.6588901357351014, + "low": 0.2826005353179266 + }, + "average_precision": { + "high": 0.7066785307286951, + "low": 0.5130564168759791 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 1.0 + }, + "mrr": { + "high": 0.9523809523809523, + "low": 0.6388888888888888 + }, + "ndcg@10": { + "high": 0.9222423078369432, + "low": 0.7102434954027206 + }, + "ndcg@20": { + "high": 0.9254379579736113, + "low": 0.7425358171776326 + }, + "ndcg@5": { + "high": 0.8388566073648231, + "low": 0.5911799981092667 + }, + "precision@10": { + "high": 0.7626798115079363, + "low": 0.5107142857142857 + }, + "precision@20": { + "high": 0.7708333333333334, + "low": 0.525 + }, + "precision@5": { + "high": 0.8, + "low": 0.5238095238095238 + }, + "recall@10": { + "high": 1.0, + "low": 0.8928571428571429 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.9111111111111111, + "low": 0.5833333333333334 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8446574402245026, + "ndcg@20": 0.8446574402245026, + "ndcg@5": 0.705601260713727, + "precision@10": 0.6666666666666666, + "precision@20": 0.6666666666666666, + "precision@5": 0.6, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.75 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.8219303906225648, + "ndcg@20": 0.8219303906225648, + "ndcg@5": 0.5087403079104241, + "precision@10": 0.5, + "precision@20": 0.5, + "precision@5": 0.4, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.4 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 1.0, + "ndcg@20": 1.0, + "ndcg@5": 1.0, + "precision@10": 1.0, + "precision@20": 1.0, + "precision@5": 1.0, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 1.0 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6309297535714575, + "ndcg@20": 0.6309297535714575, + "ndcg@5": 0.6309297535714575, + "precision@10": 0.5, + "precision@20": 0.5, + "precision@5": 0.5, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 1.0 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6108096776415199, + "ndcg@20": 0.7347887996819421, + "ndcg@5": 0.5147714448836774, + "precision@10": 0.45, + "precision@20": 0.5, + "precision@5": 0.6, + "recall@10": 0.75, + "recall@20": 1.0, + "recall@5": 0.5 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8627766724467997, + "ndcg@20": 0.8627766724467997, + "ndcg@5": 0.773049051499641, + "precision@10": 0.5714285714285714, + "precision@20": 0.5714285714285714, + "precision@5": 0.6666666666666667, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.8333333333333334 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.962987076459737, + "ndcg@20": 0.962987076459737, + "ndcg@5": 0.8421757737602428, + "precision@10": 0.625, + "precision@20": 0.625, + "precision@5": 0.8, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 0.8 + } + }, + "pooled": { + "auroc": 0.4717868338557994, + "average_precision": 0.5833356847201822, + "brier": 0.334029412618985, + "n": 51, + "positives": 29 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.4717868338557994, + "average_precision": 0.5833356847201822, + "brier": 0.334029412618985, + "n": 51, + "positives": 29 + } + } + }, + "PRIME": { + "hla": { + "HLA-A*01:01": { + "auroc": 0.8888888888888888, + "average_precision": 0.9888888888888888, + "brier": 0.7091975897099, + "n": 10, + "positives": 9 + }, + "HLA-A*02:01": { + "auroc": 0.8055555555555556, + "average_precision": 0.5888888888888888, + "brier": 0.27258663554044443, + "n": 9, + "positives": 3 + }, + "HLA-A*03:01": { + "auroc": 0.8333333333333334, + "average_precision": 0.8333333333333333, + "brier": 0.32875926336680006, + "n": 5, + "positives": 2 + }, + "HLA-A*25:01": { + "auroc": null, + "average_precision": null, + "brier": 0.000241056676, + "n": 1, + "positives": 0 + }, + "HLA-A*30:01": { + "auroc": null, + "average_precision": null, + "brier": 0.029397898114000003, + "n": 3, + "positives": 0 + }, + "HLA-A*31:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.4250756048725, + "n": 2, + "positives": 1 + }, + "HLA-A*32:01": { + "auroc": 0.5, + "average_precision": 0.8333333333333333, + "brier": 0.5460592483973333, + "n": 3, + "positives": 2 + }, + "HLA-A*66:01": { + "auroc": 0.5, + "average_precision": 0.75, + "brier": 0.42045441972875, + "n": 4, + "positives": 2 + }, + "HLA-B*07:02": { + "auroc": 0.5, + "average_precision": 0.75, + "brier": 0.42103460183875, + "n": 4, + "positives": 2 + }, + "HLA-B*08:01": { + "auroc": 0.4666666666666667, + "average_precision": 0.645, + "brier": 0.48618577261425, + "n": 8, + "positives": 5 + }, + "HLA-B*13:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.299968537172, + "n": 2, + "positives": 1 + }, + "HLA-B*18:01": { + "auroc": 0.25, + "average_precision": 0.5833333333333333, + "brier": 0.5449431063919999, + "n": 3, + "positives": 2 + }, + "HLA-B*27:05": { + "auroc": 0.7916666666666666, + "average_precision": 0.8125, + "brier": 0.32862863951029997, + "n": 10, + "positives": 4 + }, + "HLA-B*27:06": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9869568108490001, + "n": 1, + "positives": 1 + }, + "HLA-B*35:01": { + "auroc": null, + "average_precision": null, + "brier": 6.0074010000000005e-06, + "n": 1, + "positives": 0 + }, + "HLA-B*35:03": { + "auroc": 0.0, + "average_precision": 0.3333333333333333, + "brier": 0.32795754596533333, + "n": 3, + "positives": 1 + }, + "HLA-B*38:01": { + "auroc": 0.0, + "average_precision": 0.5, + "brier": 0.341915561845, + "n": 2, + "positives": 1 + }, + "HLA-B*40:01": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.526717705342375, + "n": 8, + "positives": 5 + }, + "HLA-B*44:02": { + "auroc": 0.3625, + "average_precision": 0.5592147435897437, + "brier": 0.5715760481483078, + "n": 13, + "positives": 8 + }, + "HLA-B*55:01": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.9283741232836666, + "n": 3, + "positives": 3 + }, + "HLA-B*57:01": { + "auroc": 0.5, + "average_precision": 0.6666666666666666, + "brier": 0.4117236931343333, + "n": 3, + "positives": 2 + }, + "HLA-C*03:03": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.6127647742116666, + "n": 3, + "positives": 2 + }, + "HLA-C*05:01": { + "auroc": 0.3333333333333333, + "average_precision": 0.8055555555555556, + "brier": 0.65865841489875, + "n": 4, + "positives": 3 + }, + "HLA-C*06:02": { + "auroc": 0.0, + "average_precision": 0.6791666666666667, + "brier": 0.7228783354532, + "n": 5, + "positives": 4 + }, + "HLA-C*07:01": { + "auroc": 0.44285714285714284, + "average_precision": 0.5720315398886827, + "brier": 0.56084362498675, + "n": 12, + "positives": 7 + }, + "HLA-C*07:02": { + "auroc": 1.0, + "average_precision": 1.0, + "brier": 0.70853305068425, + "n": 4, + "positives": 3 + }, + "HLA-C*12:03": { + "auroc": null, + "average_precision": 1.0, + "brier": 0.8546234622425, + "n": 2, + "positives": 2 + } + }, + "metadata": { + "score_direction": "higher", + "source": "results/raw_predictions/rcc/prime-2.0.csv", + "task": "immunogenicity", + "version": "2.0" + }, + "patient": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.7685185185185185, + "ndcg@10": 0.6881137179921539, + "ndcg@20": 0.827121210551411, + "ndcg@5": 0.6908650160295152, + "positive_bearing_patients": 9, + "precision@10": 0.5611111111111111, + "precision@20": 0.5646005983467284, + "precision@5": 0.6814814814814815, + "recall@10": 0.7203944203944204, + "recall@20": 1.0, + "recall@5": 0.47526054192720857 + }, + "patient_bootstrap_95ci": { + "auroc": { + "high": 0.6460297912268356, + "low": 0.5190087535014005 + }, + "average_precision": { + "high": 0.7184616323154791, + "low": 0.5803605807721791 + }, + "hitrate@10": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@20": { + "high": 1.0, + "low": 1.0 + }, + "hitrate@5": { + "high": 1.0, + "low": 1.0 + }, + "mrr": { + "high": 0.9074074074074074, + "low": 0.6296296296296297 + }, + "ndcg@10": { + "high": 0.7462504477591396, + "low": 0.6222563694605703 + }, + "ndcg@20": { + "high": 0.8844015947466178, + "low": 0.7626559123915735 + }, + "ndcg@5": { + "high": 0.789022108580068, + "low": 0.6109516552594605 + }, + "precision@10": { + "high": 0.6444444444444444, + "low": 0.4666666666666667 + }, + "precision@20": { + "high": 0.6469078556462458, + "low": 0.4669648367055487 + }, + "precision@5": { + "high": 0.8, + "low": 0.5703703703703704 + }, + "recall@10": { + "high": 0.821869488536155, + "low": 0.6236159211159211 + }, + "recall@20": { + "high": 1.0, + "low": 1.0 + }, + "recall@5": { + "high": 0.6257114798781466, + "low": 0.3651551226551226 + } + }, + "patient_values": { + "RCC-101": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.8047526145467747, + "ndcg@20": 0.8814274285396676, + "ndcg@5": 0.7590911424516827, + "precision@10": 0.6, + "precision@20": 0.6363636363636364, + "precision@5": 0.8, + "recall@10": 0.8571428571428571, + "recall@20": 1.0, + "recall@5": 0.5714285714285714 + }, + "RCC-102": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6240505200038379, + "ndcg@20": 0.6240505200038379, + "ndcg@5": 0.6240505200038379, + "precision@10": 0.25, + "precision@20": 0.25, + "precision@5": 0.4, + "recall@10": 1.0, + "recall@20": 1.0, + "recall@5": 1.0 + }, + "RCC-104": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.5030399532861564, + "ndcg@20": 0.7513611520351534, + "ndcg@5": 0.5147714448836774, + "precision@10": 0.5, + "precision@20": 0.5294117647058824, + "precision@5": 0.6, + "recall@10": 0.5555555555555556, + "recall@20": 1.0, + "recall@5": 0.3333333333333333 + }, + "RCC-105": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7951725670061051, + "ndcg@20": 0.9624379607364066, + "ndcg@5": 1.0, + "precision@10": 0.7, + "precision@20": 0.7368421052631579, + "precision@5": 1.0, + "recall@10": 0.5, + "recall@20": 1.0, + "recall@5": 0.35714285714285715 + }, + "RCC-106": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.5, + "ndcg@10": 0.6866059341140195, + "ndcg@20": 0.8042717497692192, + "ndcg@5": 0.6608397947263839, + "precision@10": 0.7, + "precision@20": 0.5625, + "precision@5": 0.8, + "recall@10": 0.7777777777777778, + "recall@20": 1.0, + "recall@5": 0.4444444444444444 + }, + "RCC-107": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.789149990140578, + "ndcg@20": 0.8735587987230781, + "ndcg@5": 0.6843515475204855, + "precision@10": 0.5, + "precision@20": 0.5454545454545454, + "precision@5": 0.6, + "recall@10": 0.8333333333333334, + "recall@20": 1.0, + "recall@5": 0.5 + }, + "RCC-108": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.75, + "ndcg@10": 0.6573429591465088, + "ndcg@20": 0.800344115277589, + "ndcg@5": 0.5922216874835863, + "precision@10": 0.5, + "precision@20": 0.4666666666666667, + "precision@5": 0.6, + "recall@10": 0.7142857142857143, + "recall@20": 1.0, + "recall@5": 0.42857142857142855 + }, + "RCC-109": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 0.8333333333333333, + "ndcg@10": 0.6051925781121545, + "ndcg@20": 0.8417183321921483, + "ndcg@5": 0.5742421908816506, + "precision@10": 0.6, + "precision@20": 0.6875, + "precision@5": 0.5333333333333333, + "recall@10": 0.5454545454545454, + "recall@20": 1.0, + "recall@5": 0.2424242424242424 + }, + "RCC-110": { + "hitrate@10": 1.0, + "hitrate@20": 1.0, + "hitrate@5": 1.0, + "mrr": 1.0, + "ndcg@10": 0.7277163455732502, + "ndcg@20": 0.9049208376855988, + "ndcg@5": 0.8082168163143322, + "precision@10": 0.7, + "precision@20": 0.6666666666666666, + "precision@5": 0.8, + "recall@10": 0.7, + "recall@20": 1.0, + "recall@5": 0.4 + } + }, + "pooled": { + "auroc": 0.579874213836478, + "average_precision": 0.6393093241046907, + "brier": 0.5017369006240234, + "n": 128, + "positives": 75 + }, + "study": { + "RCC_PCV_VACCINE_2025": { + "auroc": 0.579874213836478, + "average_precision": 0.6393093241046907, + "brier": 0.5017369006240234, + "n": 128, + "positives": 75 + } + } + } + }, + "paired_same_task": [ + { + "ci_high": 0.04947664122641821, + "ci_low": -0.1161501900458979, + "difference_left_minus_right": -0.028679245283018795, + "left": "BigMHC", + "left_value_common": 0.4759748427672956, + "metric": "auroc", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.5046540880503144, + "task": "immunogenicity" + }, + { + "ci_high": -0.003714798993252979, + "ci_low": -0.12680520185134853, + "difference_left_minus_right": -0.06758628692472834, + "left": "BigMHC", + "left_value_common": 0.5856631462955934, + "metric": "average_precision", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6532494332203217, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.33333333333333337, + "ci_low": 0.0, + "difference_left_minus_right": 0.11111111111111116, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.8888888888888888, + "task": "immunogenicity" + }, + { + "ci_high": 0.001898148148147924, + "ci_low": -0.18148148148148147, + "difference_left_minus_right": -0.07037037037037042, + "left": "BigMHC", + "left_value_common": 0.6518518518518518, + "metric": "mrr", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.7222222222222222, + "task": "immunogenicity" + }, + { + "ci_high": 0.012336267625158463, + "ci_low": -0.08993220825260806, + "difference_left_minus_right": -0.03903851559005356, + "left": "BigMHC", + "left_value_common": 0.6312873089172458, + "metric": "ndcg@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6703258245072994, + "task": "immunogenicity" + }, + { + "ci_high": -0.009628459310849847, + "ci_low": -0.06086790188393626, + "difference_left_minus_right": -0.03466256881045371, + "left": "BigMHC", + "left_value_common": 0.7738604771130371, + "metric": "ndcg@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.8085230459234908, + "task": "immunogenicity" + }, + { + "ci_high": 0.050409043936959864, + "ci_low": -0.1901851493238087, + "difference_left_minus_right": -0.08144831426125343, + "left": "BigMHC", + "left_value_common": 0.5325765609825637, + "metric": "ndcg@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6140248752438171, + "task": "immunogenicity" + }, + { + "ci_high": 0.0444444444444444, + "ci_low": -0.0444444444444444, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5611111111111111, + "metric": "precision@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.5611111111111111, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5646005983467284, + "metric": "precision@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.5646005983467284, + "task": "immunogenicity" + }, + { + "ci_high": 0.06666666666666665, + "ci_low": -0.2222222222222222, + "difference_left_minus_right": -0.0888888888888889, + "left": "BigMHC", + "left_value_common": 0.5333333333333333, + "metric": "precision@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.6222222222222222, + "task": "immunogenicity" + }, + { + "ci_high": 0.044973544973545, + "ci_low": -0.06878306878306872, + "difference_left_minus_right": -0.010582010582010581, + "left": "BigMHC", + "left_value_common": 0.7159050825717492, + "metric": "recall@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.7264870931537598, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.15916426166426165, + "ci_low": -0.1300316658649992, + "difference_left_minus_right": -0.003575436908770213, + "left": "BigMHC", + "left_value_common": 0.35287798621131955, + "metric": "recall@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "DeepHLApan", + "right_value_common": 0.35645342312008976, + "task": "immunogenicity" + }, + { + "ci_high": 0.16264296120925142, + "ci_low": -0.04712865259740257, + "difference_left_minus_right": 0.07210031347962381, + "left": "BigMHC", + "left_value_common": 0.5438871473354232, + "metric": "auroc", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.4717868338557994, + "task": "immunogenicity" + }, + { + "ci_high": 0.1148432249105829, + "ci_low": -0.03556215089261466, + "difference_left_minus_right": 0.048296319647384345, + "left": "BigMHC", + "left_value_common": 0.6316320043675665, + "metric": "average_precision", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.5833356847201822, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.125, + "ci_low": -0.3214285714285714, + "difference_left_minus_right": -0.04761904761904767, + "left": "BigMHC", + "left_value_common": 0.7619047619047619, + "metric": "mrr", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8095238095238095, + "task": "immunogenicity" + }, + { + "ci_high": 0.05335537459609574, + "ci_low": -0.09325707120581819, + "difference_left_minus_right": -0.010454564627022767, + "left": "BigMHC", + "left_value_common": 0.8087012940824889, + "metric": "ndcg@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8191558587095117, + "task": "immunogenicity" + }, + { + "ci_high": 0.040171356613644335, + "ci_low": -0.0973475628913677, + "difference_left_minus_right": -0.01610746654958295, + "left": "BigMHC", + "left_value_common": 0.8207596953085604, + "metric": "ndcg@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8368671618581434, + "task": "immunogenicity" + }, + { + "ci_high": 0.044278942373923366, + "ci_low": -0.09740725277182755, + "difference_left_minus_right": -0.013826215159676747, + "left": "BigMHC", + "left_value_common": 0.6969262980316332, + "metric": "ndcg@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7107525131913099, + "task": "immunogenicity" + }, + { + "ci_high": 0.022222222222222143, + "ci_low": 0.0, + "difference_left_minus_right": 0.0071428571428571175, + "left": "BigMHC", + "left_value_common": 0.6232993197278911, + "metric": "precision@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.616156462585034, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.6232993197278911, + "metric": "precision@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6232993197278911, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.6523809523809524, + "metric": "precision@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6523809523809524, + "task": "immunogenicity" + }, + { + "ci_high": 0.03703703703703709, + "ci_low": 0.0, + "difference_left_minus_right": 0.011904761904761862, + "left": "BigMHC", + "left_value_common": 0.9761904761904762, + "metric": "recall@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.9642857142857143, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.7547619047619047, + "metric": "recall@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7547619047619047, + "task": "immunogenicity" + }, + { + "ci_high": -0.015272674169330787, + "ci_low": -0.1891589126906106, + "difference_left_minus_right": -0.10389937106918234, + "left": "BigMHC", + "left_value_common": 0.4759748427672956, + "metric": "auroc", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.579874213836478, + "task": "immunogenicity" + }, + { + "ci_high": 0.01740314444245059, + "ci_low": -0.12351507022065718, + "difference_left_minus_right": -0.053646177809097306, + "left": "BigMHC", + "left_value_common": 0.5856631462955934, + "metric": "average_precision", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6393093241046907, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.07407407407407407, + "ci_low": -0.30370370370370364, + "difference_left_minus_right": -0.1166666666666667, + "left": "BigMHC", + "left_value_common": 0.6518518518518518, + "metric": "mrr", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7685185185185185, + "task": "immunogenicity" + }, + { + "ci_high": 0.03017239737900372, + "ci_low": -0.14483158331026183, + "difference_left_minus_right": -0.0568264090749081, + "left": "BigMHC", + "left_value_common": 0.6312873089172458, + "metric": "ndcg@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6881137179921539, + "task": "immunogenicity" + }, + { + "ci_high": 0.008075273494052092, + "ci_low": -0.11794338501591053, + "difference_left_minus_right": -0.0532607334383739, + "left": "BigMHC", + "left_value_common": 0.7738604771130371, + "metric": "ndcg@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.827121210551411, + "task": "immunogenicity" + }, + { + "ci_high": -0.023271734532436354, + "ci_low": -0.2990950049162747, + "difference_left_minus_right": -0.15828845504695155, + "left": "BigMHC", + "left_value_common": 0.5325765609825637, + "metric": "ndcg@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6908650160295152, + "task": "immunogenicity" + }, + { + "ci_high": 0.06666666666666676, + "ci_low": -0.06666666666666665, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5611111111111111, + "metric": "precision@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5611111111111111, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 0.5646005983467284, + "metric": "precision@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5646005983467284, + "task": "immunogenicity" + }, + { + "ci_high": -0.0444444444444444, + "ci_low": -0.2594444444444445, + "difference_left_minus_right": -0.14814814814814814, + "left": "BigMHC", + "left_value_common": 0.5333333333333333, + "metric": "precision@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6814814814814815, + "task": "immunogenicity" + }, + { + "ci_high": 0.07215007215007219, + "ci_low": -0.08113275613275624, + "difference_left_minus_right": -0.004489337822671136, + "left": "BigMHC", + "left_value_common": 0.7159050825717492, + "metric": "recall@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7203944203944204, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "BigMHC", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": -0.03366442199775537, + "ci_low": -0.23490259740259736, + "difference_left_minus_right": -0.12238255571588902, + "left": "BigMHC", + "left_value_common": 0.35287798621131955, + "metric": "recall@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.47526054192720857, + "task": "immunogenicity" + }, + { + "ci_high": 0.13381416410650276, + "ci_low": -0.17358745742336204, + "difference_left_minus_right": -0.010971786833855801, + "left": "DeepHLApan", + "left_value_common": 0.4608150470219436, + "metric": "auroc", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.4717868338557994, + "task": "immunogenicity" + }, + { + "ci_high": 0.12016090914540065, + "ci_low": -0.0844896794670285, + "difference_left_minus_right": 0.03270811775896276, + "left": "DeepHLApan", + "left_value_common": 0.6160438024791449, + "metric": "average_precision", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.5833356847201822, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.20020833333333227, + "ci_low": -0.09999999999999998, + "difference_left_minus_right": 0.0357142857142857, + "left": "DeepHLApan", + "left_value_common": 0.8452380952380952, + "metric": "mrr", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8095238095238095, + "task": "immunogenicity" + }, + { + "ci_high": 0.12901205374691363, + "ci_low": -0.04765365130313093, + "difference_left_minus_right": 0.03239542824625208, + "left": "DeepHLApan", + "left_value_common": 0.8515512869557638, + "metric": "ndcg@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8191558587095117, + "task": "immunogenicity" + }, + { + "ci_high": 0.08191660986424498, + "ci_low": -0.05020882164937479, + "difference_left_minus_right": 0.014684125097620404, + "left": "DeepHLApan", + "left_value_common": 0.8515512869557638, + "metric": "ndcg@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.8368671618581434, + "task": "immunogenicity" + }, + { + "ci_high": 0.11382296763686184, + "ci_low": -0.0833180287588855, + "difference_left_minus_right": 0.015728646900082577, + "left": "DeepHLApan", + "left_value_common": 0.7264811600913925, + "metric": "ndcg@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7107525131913099, + "task": "immunogenicity" + }, + { + "ci_high": 0.06428571428571428, + "ci_low": 0.0, + "difference_left_minus_right": 0.021428571428571463, + "left": "DeepHLApan", + "left_value_common": 0.6375850340136054, + "metric": "precision@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.616156462585034, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.6232993197278911, + "metric": "precision@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6232993197278911, + "task": "immunogenicity" + }, + { + "ci_high": 0.08571428571428574, + "ci_low": -0.08000000000000007, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.6523809523809524, + "metric": "precision@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.6523809523809524, + "task": "immunogenicity" + }, + { + "ci_high": 0.1071428571428571, + "ci_low": 0.0, + "difference_left_minus_right": 0.0357142857142857, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "recall@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.9642857142857143, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.08571428571428574, + "ci_low": -0.08000000000000007, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.7547619047619047, + "metric": "recall@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "DeepImmuno-CNN", + "right_value_common": 0.7547619047619047, + "task": "immunogenicity" + }, + { + "ci_high": 0.04368853820754026, + "ci_low": -0.195250402792696, + "difference_left_minus_right": -0.07522012578616355, + "left": "DeepHLApan", + "left_value_common": 0.5046540880503144, + "metric": "auroc", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.579874213836478, + "task": "immunogenicity" + }, + { + "ci_high": 0.09172009118667658, + "ci_low": -0.09238799817801838, + "difference_left_minus_right": 0.013940109115631039, + "left": "DeepHLApan", + "left_value_common": 0.6532494332203217, + "metric": "average_precision", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6393093241046907, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": -0.33333333333333337, + "difference_left_minus_right": -0.11111111111111116, + "left": "DeepHLApan", + "left_value_common": 0.8888888888888888, + "metric": "hitrate@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.12962962962962965, + "ci_low": -0.2314814814814815, + "difference_left_minus_right": -0.04629629629629628, + "left": "DeepHLApan", + "left_value_common": 0.7222222222222222, + "metric": "mrr", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7685185185185185, + "task": "immunogenicity" + }, + { + "ci_high": 0.08331803966539872, + "ci_low": -0.12058836256824594, + "difference_left_minus_right": -0.017787893484854544, + "left": "DeepHLApan", + "left_value_common": 0.6703258245072994, + "metric": "ndcg@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6881137179921539, + "task": "immunogenicity" + }, + { + "ci_high": 0.04597761031229554, + "ci_low": -0.0910240490275227, + "difference_left_minus_right": -0.01859816462792019, + "left": "DeepHLApan", + "left_value_common": 0.8085230459234908, + "metric": "ndcg@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.827121210551411, + "task": "immunogenicity" + }, + { + "ci_high": 0.09313139286054935, + "ci_low": -0.26593716874969686, + "difference_left_minus_right": -0.07684014078569812, + "left": "DeepHLApan", + "left_value_common": 0.6140248752438171, + "metric": "ndcg@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6908650160295152, + "task": "immunogenicity" + }, + { + "ci_high": 0.06666666666666676, + "ci_low": -0.06666666666666665, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.5611111111111111, + "metric": "precision@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5611111111111111, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 0.5646005983467284, + "metric": "precision@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.5646005983467284, + "task": "immunogenicity" + }, + { + "ci_high": 0.0888888888888889, + "ci_low": -0.20000000000000007, + "difference_left_minus_right": -0.059259259259259234, + "left": "DeepHLApan", + "left_value_common": 0.6222222222222222, + "metric": "precision@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.6814814814814815, + "task": "immunogenicity" + }, + { + "ci_high": 0.07744107744107742, + "ci_low": -0.07319223985890644, + "difference_left_minus_right": 0.006092672759339446, + "left": "DeepHLApan", + "left_value_common": 0.7264870931537598, + "metric": "recall@10", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.7203944203944204, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepHLApan", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.048110068943402265, + "ci_low": -0.36333172999839664, + "difference_left_minus_right": -0.1188071188071188, + "left": "DeepHLApan", + "left_value_common": 0.35645342312008976, + "metric": "recall@5", + "n_common": 128, + "patients_common": 9, + "positives_common": 75, + "right": "PRIME", + "right_value_common": 0.47526054192720857, + "task": "immunogenicity" + }, + { + "ci_high": 0.04963112522686011, + "ci_low": -0.3485843515037594, + "difference_left_minus_right": -0.15047021943573663, + "left": "DeepImmuno-CNN", + "left_value_common": 0.4717868338557994, + "metric": "auroc", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.622257053291536, + "task": "immunogenicity" + }, + { + "ci_high": 0.03802709044411525, + "ci_low": -0.1543269255374634, + "difference_left_minus_right": -0.04858767874061665, + "left": "DeepImmuno-CNN", + "left_value_common": 0.5833356847201822, + "metric": "average_precision", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6319233634607988, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "hitrate@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "hitrate@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "hitrate@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.20000000000000007, + "ci_low": -0.09999999999999998, + "difference_left_minus_right": 0.0357142857142857, + "left": "DeepImmuno-CNN", + "left_value_common": 0.8095238095238095, + "metric": "mrr", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.7738095238095238, + "task": "immunogenicity" + }, + { + "ci_high": 0.05146739445932063, + "ci_low": -0.0835917238927503, + "difference_left_minus_right": -0.007970910076125937, + "left": "DeepImmuno-CNN", + "left_value_common": 0.8191558587095117, + "metric": "ndcg@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.8271267687856376, + "task": "immunogenicity" + }, + { + "ci_high": 0.05207977361323347, + "ci_low": -0.03209450981710745, + "difference_left_minus_right": 0.009740393072505737, + "left": "DeepImmuno-CNN", + "left_value_common": 0.8368671618581434, + "metric": "ndcg@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.8271267687856376, + "task": "immunogenicity" + }, + { + "ci_high": 0.0826002139424956, + "ci_low": -0.08016590563417962, + "difference_left_minus_right": 0.0036628776135311236, + "left": "DeepImmuno-CNN", + "left_value_common": 0.7107525131913099, + "metric": "ndcg@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.7070896355777788, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": -0.06428571428571428, + "difference_left_minus_right": -0.021428571428571463, + "left": "DeepImmuno-CNN", + "left_value_common": 0.616156462585034, + "metric": "precision@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6375850340136054, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 0.6232993197278911, + "metric": "precision@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6232993197278911, + "task": "immunogenicity" + }, + { + "ci_high": 0.09333333333333338, + "ci_low": -0.11428571428571432, + "difference_left_minus_right": -0.00952380952380949, + "left": "DeepImmuno-CNN", + "left_value_common": 0.6523809523809524, + "metric": "precision@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.6619047619047619, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": -0.1071428571428571, + "difference_left_minus_right": -0.0357142857142857, + "left": "DeepImmuno-CNN", + "left_value_common": 0.9642857142857143, + "metric": "recall@10", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.0, + "ci_low": 0.0, + "difference_left_minus_right": 0.0, + "left": "DeepImmuno-CNN", + "left_value_common": 1.0, + "metric": "recall@20", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 1.0, + "task": "immunogenicity" + }, + { + "ci_high": 0.10000000000000009, + "ci_low": -0.12857142857142856, + "difference_left_minus_right": -0.011904761904761862, + "left": "DeepImmuno-CNN", + "left_value_common": 0.7547619047619047, + "metric": "recall@5", + "n_common": 51, + "patients_common": 9, + "positives_common": 29, + "right": "PRIME", + "right_value_common": 0.7666666666666666, + "task": "immunogenicity" + } + ] +} diff --git a/results/analysis/rcc/missingness.csv b/results/analysis/rcc/missingness.csv new file mode 100644 index 0000000..2383eb5 --- /dev/null +++ b/results/analysis/rcc/missingness.csv @@ -0,0 +1,10 @@ +predictor,status,count +BigMHC,predicted,128 +BigMHC,unsupported_hla,1 +DeepHLApan,predicted,128 +DeepHLApan,unsupported_hla,1 +DeepImmuno-CNN,predicted,51 +DeepImmuno-CNN,unsupported_hla,29 +DeepImmuno-CNN,unsupported_length,49 +PRIME,predicted,128 +PRIME,unsupported_hla,1 diff --git a/results/analysis/rcc/paired_differences.csv b/results/analysis/rcc/paired_differences.csv new file mode 100644 index 0000000..255dbda --- /dev/null +++ b/results/analysis/rcc/paired_differences.csv @@ -0,0 +1,91 @@ +left,right,task,metric,n_common,positives_common,patients_common,left_value_common,right_value_common,difference_left_minus_right,ci_low,ci_high +BigMHC,DeepHLApan,immunogenicity,auroc,128,75,9,0.4759748427672956,0.5046540880503144,-0.028679245283018795,-0.1161501900458979,0.04947664122641821 +BigMHC,DeepHLApan,immunogenicity,average_precision,128,75,9,0.5856631462955934,0.6532494332203217,-0.06758628692472834,-0.12680520185134853,-0.003714798993252979 +BigMHC,DeepHLApan,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,hitrate@5,128,75,9,1.0,0.8888888888888888,0.11111111111111116,0.0,0.33333333333333337 +BigMHC,DeepHLApan,immunogenicity,mrr,128,75,9,0.6518518518518518,0.7222222222222222,-0.07037037037037042,-0.18148148148148147,0.001898148148147924 +BigMHC,DeepHLApan,immunogenicity,ndcg@10,128,75,9,0.6312873089172458,0.6703258245072994,-0.03903851559005356,-0.08993220825260806,0.012336267625158463 +BigMHC,DeepHLApan,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.8085230459234908,-0.03466256881045371,-0.06086790188393626,-0.009628459310849847 +BigMHC,DeepHLApan,immunogenicity,ndcg@5,128,75,9,0.5325765609825637,0.6140248752438171,-0.08144831426125343,-0.1901851493238087,0.050409043936959864 +BigMHC,DeepHLApan,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.0444444444444444,0.0444444444444444 +BigMHC,DeepHLApan,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,precision@5,128,75,9,0.5333333333333333,0.6222222222222222,-0.0888888888888889,-0.2222222222222222,0.06666666666666665 +BigMHC,DeepHLApan,immunogenicity,recall@10,128,75,9,0.7159050825717492,0.7264870931537598,-0.010582010582010581,-0.06878306878306872,0.044973544973545 +BigMHC,DeepHLApan,immunogenicity,recall@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepHLApan,immunogenicity,recall@5,128,75,9,0.35287798621131955,0.35645342312008976,-0.003575436908770213,-0.1300316658649992,0.15916426166426165 +BigMHC,DeepImmuno-CNN,immunogenicity,auroc,51,29,9,0.5438871473354232,0.4717868338557994,0.07210031347962381,-0.04712865259740257,0.16264296120925142 +BigMHC,DeepImmuno-CNN,immunogenicity,average_precision,51,29,9,0.6316320043675665,0.5833356847201822,0.048296319647384345,-0.03556215089261466,0.1148432249105829 +BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@10,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,mrr,51,29,9,0.7619047619047619,0.8095238095238095,-0.04761904761904767,-0.3214285714285714,0.125 +BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@10,51,29,9,0.8087012940824889,0.8191558587095117,-0.010454564627022767,-0.09325707120581819,0.05335537459609574 +BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8207596953085604,0.8368671618581434,-0.01610746654958295,-0.0973475628913677,0.040171356613644335 +BigMHC,DeepImmuno-CNN,immunogenicity,ndcg@5,51,29,9,0.6969262980316332,0.7107525131913099,-0.013826215159676747,-0.09740725277182755,0.044278942373923366 +BigMHC,DeepImmuno-CNN,immunogenicity,precision@10,51,29,9,0.6232993197278911,0.616156462585034,0.0071428571428571175,0.0,0.022222222222222143 +BigMHC,DeepImmuno-CNN,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,precision@5,51,29,9,0.6523809523809524,0.6523809523809524,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,recall@10,51,29,9,0.9761904761904762,0.9642857142857143,0.011904761904761862,0.0,0.03703703703703709 +BigMHC,DeepImmuno-CNN,immunogenicity,recall@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,DeepImmuno-CNN,immunogenicity,recall@5,51,29,9,0.7547619047619047,0.7547619047619047,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,auroc,128,75,9,0.4759748427672956,0.579874213836478,-0.10389937106918234,-0.1891589126906106,-0.015272674169330787 +BigMHC,PRIME,immunogenicity,average_precision,128,75,9,0.5856631462955934,0.6393093241046907,-0.053646177809097306,-0.12351507022065718,0.01740314444245059 +BigMHC,PRIME,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,hitrate@5,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,mrr,128,75,9,0.6518518518518518,0.7685185185185185,-0.1166666666666667,-0.30370370370370364,0.07407407407407407 +BigMHC,PRIME,immunogenicity,ndcg@10,128,75,9,0.6312873089172458,0.6881137179921539,-0.0568264090749081,-0.14483158331026183,0.03017239737900372 +BigMHC,PRIME,immunogenicity,ndcg@20,128,75,9,0.7738604771130371,0.827121210551411,-0.0532607334383739,-0.11794338501591053,0.008075273494052092 +BigMHC,PRIME,immunogenicity,ndcg@5,128,75,9,0.5325765609825637,0.6908650160295152,-0.15828845504695155,-0.2990950049162747,-0.023271734532436354 +BigMHC,PRIME,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.06666666666666665,0.06666666666666676 +BigMHC,PRIME,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,precision@5,128,75,9,0.5333333333333333,0.6814814814814815,-0.14814814814814814,-0.2594444444444445,-0.0444444444444444 +BigMHC,PRIME,immunogenicity,recall@10,128,75,9,0.7159050825717492,0.7203944203944204,-0.004489337822671136,-0.08113275613275624,0.07215007215007219 +BigMHC,PRIME,immunogenicity,recall@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +BigMHC,PRIME,immunogenicity,recall@5,128,75,9,0.35287798621131955,0.47526054192720857,-0.12238255571588902,-0.23490259740259736,-0.03366442199775537 +DeepHLApan,DeepImmuno-CNN,immunogenicity,auroc,51,29,9,0.4608150470219436,0.4717868338557994,-0.010971786833855801,-0.17358745742336204,0.13381416410650276 +DeepHLApan,DeepImmuno-CNN,immunogenicity,average_precision,51,29,9,0.6160438024791449,0.5833356847201822,0.03270811775896276,-0.0844896794670285,0.12016090914540065 +DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@10,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,mrr,51,29,9,0.8452380952380952,0.8095238095238095,0.0357142857142857,-0.09999999999999998,0.20020833333333227 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@10,51,29,9,0.8515512869557638,0.8191558587095117,0.03239542824625208,-0.04765365130313093,0.12901205374691363 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@20,51,29,9,0.8515512869557638,0.8368671618581434,0.014684125097620404,-0.05020882164937479,0.08191660986424498 +DeepHLApan,DeepImmuno-CNN,immunogenicity,ndcg@5,51,29,9,0.7264811600913925,0.7107525131913099,0.015728646900082577,-0.0833180287588855,0.11382296763686184 +DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@10,51,29,9,0.6375850340136054,0.616156462585034,0.021428571428571463,0.0,0.06428571428571428 +DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,precision@5,51,29,9,0.6523809523809524,0.6523809523809524,0.0,-0.08000000000000007,0.08571428571428574 +DeepHLApan,DeepImmuno-CNN,immunogenicity,recall@10,51,29,9,1.0,0.9642857142857143,0.0357142857142857,0.0,0.1071428571428571 +DeepHLApan,DeepImmuno-CNN,immunogenicity,recall@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,DeepImmuno-CNN,immunogenicity,recall@5,51,29,9,0.7547619047619047,0.7547619047619047,0.0,-0.08000000000000007,0.08571428571428574 +DeepHLApan,PRIME,immunogenicity,auroc,128,75,9,0.5046540880503144,0.579874213836478,-0.07522012578616355,-0.195250402792696,0.04368853820754026 +DeepHLApan,PRIME,immunogenicity,average_precision,128,75,9,0.6532494332203217,0.6393093241046907,0.013940109115631039,-0.09238799817801838,0.09172009118667658 +DeepHLApan,PRIME,immunogenicity,hitrate@10,128,75,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,hitrate@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,hitrate@5,128,75,9,0.8888888888888888,1.0,-0.11111111111111116,-0.33333333333333337,0.0 +DeepHLApan,PRIME,immunogenicity,mrr,128,75,9,0.7222222222222222,0.7685185185185185,-0.04629629629629628,-0.2314814814814815,0.12962962962962965 +DeepHLApan,PRIME,immunogenicity,ndcg@10,128,75,9,0.6703258245072994,0.6881137179921539,-0.017787893484854544,-0.12058836256824594,0.08331803966539872 +DeepHLApan,PRIME,immunogenicity,ndcg@20,128,75,9,0.8085230459234908,0.827121210551411,-0.01859816462792019,-0.0910240490275227,0.04597761031229554 +DeepHLApan,PRIME,immunogenicity,ndcg@5,128,75,9,0.6140248752438171,0.6908650160295152,-0.07684014078569812,-0.26593716874969686,0.09313139286054935 +DeepHLApan,PRIME,immunogenicity,precision@10,128,75,9,0.5611111111111111,0.5611111111111111,0.0,-0.06666666666666665,0.06666666666666676 +DeepHLApan,PRIME,immunogenicity,precision@20,128,75,9,0.5646005983467284,0.5646005983467284,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,precision@5,128,75,9,0.6222222222222222,0.6814814814814815,-0.059259259259259234,-0.20000000000000007,0.0888888888888889 +DeepHLApan,PRIME,immunogenicity,recall@10,128,75,9,0.7264870931537598,0.7203944203944204,0.006092672759339446,-0.07319223985890644,0.07744107744107742 +DeepHLApan,PRIME,immunogenicity,recall@20,128,75,9,1.0,1.0,0.0,0.0,0.0 +DeepHLApan,PRIME,immunogenicity,recall@5,128,75,9,0.35645342312008976,0.47526054192720857,-0.1188071188071188,-0.36333172999839664,0.048110068943402265 +DeepImmuno-CNN,PRIME,immunogenicity,auroc,51,29,9,0.4717868338557994,0.622257053291536,-0.15047021943573663,-0.3485843515037594,0.04963112522686011 +DeepImmuno-CNN,PRIME,immunogenicity,average_precision,51,29,9,0.5833356847201822,0.6319233634607988,-0.04858767874061665,-0.1543269255374634,0.03802709044411525 +DeepImmuno-CNN,PRIME,immunogenicity,hitrate@10,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,hitrate@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,hitrate@5,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,mrr,51,29,9,0.8095238095238095,0.7738095238095238,0.0357142857142857,-0.09999999999999998,0.20000000000000007 +DeepImmuno-CNN,PRIME,immunogenicity,ndcg@10,51,29,9,0.8191558587095117,0.8271267687856376,-0.007970910076125937,-0.0835917238927503,0.05146739445932063 +DeepImmuno-CNN,PRIME,immunogenicity,ndcg@20,51,29,9,0.8368671618581434,0.8271267687856376,0.009740393072505737,-0.03209450981710745,0.05207977361323347 +DeepImmuno-CNN,PRIME,immunogenicity,ndcg@5,51,29,9,0.7107525131913099,0.7070896355777788,0.0036628776135311236,-0.08016590563417962,0.0826002139424956 +DeepImmuno-CNN,PRIME,immunogenicity,precision@10,51,29,9,0.616156462585034,0.6375850340136054,-0.021428571428571463,-0.06428571428571428,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,precision@20,51,29,9,0.6232993197278911,0.6232993197278911,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,precision@5,51,29,9,0.6523809523809524,0.6619047619047619,-0.00952380952380949,-0.11428571428571432,0.09333333333333338 +DeepImmuno-CNN,PRIME,immunogenicity,recall@10,51,29,9,0.9642857142857143,1.0,-0.0357142857142857,-0.1071428571428571,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,recall@20,51,29,9,1.0,1.0,0.0,0.0,0.0 +DeepImmuno-CNN,PRIME,immunogenicity,recall@5,51,29,9,0.7547619047619047,0.7666666666666666,-0.011904761904761862,-0.12857142857142856,0.10000000000000009 diff --git a/results/analysis/stability/analysis_metadata.json b/results/analysis/stability/analysis_metadata.json new file mode 100644 index 0000000..bad202a --- /dev/null +++ b/results/analysis/stability/analysis_metadata.json @@ -0,0 +1,68 @@ +{ + "analysis_type": "exploratory_descriptive_heterogeneity", + "bootstrap": 2000, + "seed": 20260820, + "ks": [ + 1, + 5, + 10, + 20 + ], + "coverage_thresholds": [ + 0.5, + 0.8, + 0.95 + ], + "datasets": [ + "improve_benchmark", + "zhao_vaccine_benchmark", + "rcc_vaccine_benchmark" + ], + "predictors": { + "BigMHC": { + "task": "immunogenicity", + "version": "v1.0", + "status": { + "predicted": 128, + "unsupported_hla": 1 + } + }, + "MHCflurry": { + "task": "presentation", + "version": "2.2.1", + "status": { + "predicted": 2315 + } + }, + "PRIME": { + "task": "immunogenicity", + "version": "2.0", + "status": { + "predicted": 128, + "unsupported_hla": 1 + } + }, + "DeepHLApan": { + "task": "immunogenicity", + "version": "1.1.1@ac1f4beb", + "status": { + "predicted": 128, + "unsupported_hla": 1 + } + }, + "DeepImmuno-CNN": { + "task": "immunogenicity", + "version": "1.0@df42ac5b", + "status": { + "predicted": 51, + "unsupported_length": 49, + "unsupported_hla": 29 + } + } + }, + "limitations": [ + "fixed pretrained scores; no causal inference", + "model-selection probabilities are conditional on observed patient samples", + "coverage strategies are descriptive and not multiplicity-adjusted" + ] +} diff --git a/results/analysis/stability/dataset_predictor_metric_matrix.csv b/results/analysis/stability/dataset_predictor_metric_matrix.csv new file mode 100644 index 0000000..d12f058 --- /dev/null +++ b/results/analysis/stability/dataset_predictor_metric_matrix.csv @@ -0,0 +1,61 @@ +coverage,dataset,metric,n,predictor,value +1.0,improve_benchmark,AUROC,17475,BigMHC,0.5458288293413741 +1.0,improve_benchmark,Recall@1,17475,BigMHC,0.01890873015873016 +1.0,improve_benchmark,Recall@5,17475,BigMHC,0.038045634920634924 +1.0,improve_benchmark,Recall@10,17475,BigMHC,0.07884220626867686 +1.0,improve_benchmark,Recall@20,17475,BigMHC,0.14580332700248363 +1.0,improve_benchmark,AUROC,17475,MHCflurry,0.5367554822274058 +1.0,improve_benchmark,Recall@1,17475,MHCflurry,0.009533730158730158 +1.0,improve_benchmark,Recall@5,17475,MHCflurry,0.04661971434997751 +1.0,improve_benchmark,Recall@10,17475,MHCflurry,0.08175583503756877 +1.0,improve_benchmark,Recall@20,17475,MHCflurry,0.20210686751655038 +1.0,improve_benchmark,AUROC,17475,PRIME,0.5969085863470571 +1.0,improve_benchmark,Recall@1,17475,PRIME,0.008503455608718766 +1.0,improve_benchmark,Recall@5,17475,PRIME,0.0589177645722564 +1.0,improve_benchmark,Recall@10,17475,PRIME,0.13437829088804587 +1.0,improve_benchmark,Recall@20,17475,PRIME,0.26004746077835117 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,BigMHC,0.5423340961098398 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,BigMHC,0.39791666666666664 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,BigMHC,0.9541666666666666 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,BigMHC,0.9875 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,BigMHC,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,DeepHLApan,0.5072058035931643 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,DeepHLApan,0.35625 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,DeepHLApan,0.9249999999999998 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,DeepHLApan,1.0 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,DeepHLApan,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,DeepImmuno-CNN,0.5257315351282925 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,DeepImmuno-CNN,0.3770833333333333 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,DeepImmuno-CNN,0.9249999999999998 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,DeepImmuno-CNN,0.9875 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,DeepImmuno-CNN,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,MHCflurry,0.5430968726163234 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,MHCflurry,0.3572916666666667 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,MHCflurry,0.9322916666666666 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,MHCflurry,1.0 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,MHCflurry,1.0 +0.43844492440604754,zhao_vaccine_benchmark,AUROC,1015,PRIME,0.5365240112306668 +0.43844492440604754,zhao_vaccine_benchmark,Recall@1,1015,PRIME,0.37083333333333335 +0.43844492440604754,zhao_vaccine_benchmark,Recall@5,1015,PRIME,0.9302083333333334 +0.43844492440604754,zhao_vaccine_benchmark,Recall@10,1015,PRIME,1.0 +0.43844492440604754,zhao_vaccine_benchmark,Recall@20,1015,PRIME,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,BigMHC,0.5438871473354232 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,BigMHC,0.15952380952380954 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,BigMHC,0.7785714285714286 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,BigMHC,0.9761904761904762 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,BigMHC,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,DeepHLApan,0.4608150470219436 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,DeepHLApan,0.1880952380952381 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,DeepHLApan,0.7785714285714285 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,DeepHLApan,1.0 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,DeepHLApan,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,DeepImmuno-CNN,0.4717868338557994 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,DeepImmuno-CNN,0.16428571428571428 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,DeepImmuno-CNN,0.7785714285714286 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,DeepImmuno-CNN,0.9761904761904762 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,DeepImmuno-CNN,1.0 +0.3953488372093023,rcc_vaccine_benchmark,AUROC,51,PRIME,0.622257053291536 +0.3953488372093023,rcc_vaccine_benchmark,Recall@1,51,PRIME,0.15952380952380954 +0.3953488372093023,rcc_vaccine_benchmark,Recall@5,51,PRIME,0.7785714285714285 +0.3953488372093023,rcc_vaccine_benchmark,Recall@10,51,PRIME,1.0 +0.3953488372093023,rcc_vaccine_benchmark,Recall@20,51,PRIME,1.0 diff --git a/results/analysis/stability/endpoint_domain_auroc.png b/results/analysis/stability/endpoint_domain_auroc.png new file mode 100644 index 0000000..75d60ee Binary files /dev/null and b/results/analysis/stability/endpoint_domain_auroc.png differ diff --git a/results/analysis/stability/endpoint_domain_auroc.svg b/results/analysis/stability/endpoint_domain_auroc.svg new file mode 100644 index 0000000..fb52c89 --- /dev/null +++ b/results/analysis/stability/endpoint_domain_auroc.svg @@ -0,0 +1,43 @@ + + +Exploratory AUROC by endpoint/domain and predictor +improve_benchmark +rcc_vaccine_benchmark +zhao_vaccine_benchmark +BigMHC + +0.546 + +0.544 + +0.542 +DeepHLApan + +NA + +0.461 + +0.507 +DeepImmuno-CNN + +NA + +0.472 + +0.526 +MHCflurry + +0.537 + +NA + +0.543 +PRIME + +0.597 + +0.622 + +0.537 +Fixed pretrained scores; descriptive heterogeneity only; endpoint/domain differences are not causal estimates. + diff --git a/results/analysis/stability/endpoint_domain_metadata.csv b/results/analysis/stability/endpoint_domain_metadata.csv new file mode 100644 index 0000000..fac0c3d --- /dev/null +++ b/results/analysis/stability/endpoint_domain_metadata.csv @@ -0,0 +1,4 @@ +clinical_context,dataset,endpoint,n_patients,n_records,n_studies +PD-L1 checkpoint inhibition,improve_benchmark,DNA-barcoded peptide-MHC multimer T-cell recognition,70,17475,3 +personalized_peptide_pulsed_DC_vaccine,zhao_vaccine_benchmark,post_vaccination_IFNG_ELISPOT,352,2315,1 +personalized_RCC_peptide_vaccine,rcc_vaccine_benchmark,post_vaccine_invitro_IFNG_ELISPOT,9,129,1 diff --git a/results/analysis/stability/leave_one_domain_out.csv b/results/analysis/stability/leave_one_domain_out.csv new file mode 100644 index 0000000..6e47def --- /dev/null +++ b/results/analysis/stability/leave_one_domain_out.csv @@ -0,0 +1,19 @@ +analysis_type,dataset,held_out_domain,metric,n,patients,predictor,value +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-basket,AUROC,12115,50,BigMHC,0.510179883078191 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-basket,AUROC,12115,50,MHCflurry,0.5300323841168303 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-basket,AUROC,12115,50,PRIME,0.5741732163043727 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-mUC,AUROC,11240,46,BigMHC,0.5636546281868342 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-mUC,AUROC,11240,46,MHCflurry,0.5404874933634612 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-mUC,AUROC,11240,46,PRIME,0.6117649277495425 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-melanoma,AUROC,11595,44,BigMHC,0.5708696661696276 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-melanoma,AUROC,11595,44,MHCflurry,0.5392223509184932 +descriptive_leave_one_domain_out,improve_benchmark,IMPROVE-melanoma,AUROC,11595,44,PRIME,0.6088626172526672 +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,BigMHC, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,DeepHLApan, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,DeepImmuno-CNN, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,MHCflurry, +descriptive_leave_one_domain_out,zhao_vaccine_benchmark,ZHAO_DC_VACCINE_2026,AUROC,0,0,PRIME, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,BigMHC, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,DeepHLApan, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,DeepImmuno-CNN, +descriptive_leave_one_domain_out,rcc_vaccine_benchmark,RCC_PCV_VACCINE_2025,AUROC,0,0,PRIME, diff --git a/results/analysis/stability/model_selection_first_probability.csv b/results/analysis/stability/model_selection_first_probability.csv new file mode 100644 index 0000000..a2f916d --- /dev/null +++ b/results/analysis/stability/model_selection_first_probability.csv @@ -0,0 +1,13 @@ +dataset,metric,predictor,probability_first,task +improve_benchmark,Recall@5,BigMHC,0.0815,immunogenicity +improve_benchmark,Recall@5,PRIME,0.9185,immunogenicity +improve_benchmark,Recall@5,MHCflurry,1.0,presentation +zhao_vaccine_benchmark,Recall@5,BigMHC,0.001,immunogenicity +zhao_vaccine_benchmark,Recall@5,DeepHLApan,0.0,immunogenicity +zhao_vaccine_benchmark,Recall@5,DeepImmuno-CNN,0.999,immunogenicity +zhao_vaccine_benchmark,Recall@5,PRIME,0.0,immunogenicity +zhao_vaccine_benchmark,Recall@5,MHCflurry,1.0,presentation +rcc_vaccine_benchmark,Recall@5,BigMHC,0.0,immunogenicity +rcc_vaccine_benchmark,Recall@5,DeepHLApan,0.0,immunogenicity +rcc_vaccine_benchmark,Recall@5,DeepImmuno-CNN,0.99,immunogenicity +rcc_vaccine_benchmark,Recall@5,PRIME,0.01,immunogenicity diff --git a/results/analysis/stability/rank_stability.csv b/results/analysis/stability/rank_stability.csv new file mode 100644 index 0000000..0b9c38c --- /dev/null +++ b/results/analysis/stability/rank_stability.csv @@ -0,0 +1,96 @@ +dataset,left,metric,n,right,value +improve_benchmark,BigMHC,record_score_spearman,17475,MHCflurry,0.4318233968048541 +improve_benchmark,BigMHC,patient_Recall@1_difference,17475,MHCflurry,0.009375000000000001 +improve_benchmark,BigMHC,patient_Recall@5_difference,17475,MHCflurry,-0.008574079429342588 +improve_benchmark,BigMHC,patient_Recall@10_difference,17475,MHCflurry,-0.0029136287688919377 +improve_benchmark,BigMHC,patient_Recall@20_difference,17475,MHCflurry,-0.0563035405140668 +improve_benchmark,BigMHC,record_score_spearman,17475,PRIME,0.40658523103185173 +improve_benchmark,BigMHC,patient_Recall@1_difference,17475,PRIME,0.010405274550011393 +improve_benchmark,BigMHC,patient_Recall@5_difference,17475,PRIME,-0.020872129651621482 +improve_benchmark,BigMHC,patient_Recall@10_difference,17475,PRIME,-0.055536084619369014 +improve_benchmark,BigMHC,patient_Recall@20_difference,17475,PRIME,-0.11424413377586748 +improve_benchmark,MHCflurry,record_score_spearman,17475,PRIME,0.6508621316344753 +improve_benchmark,MHCflurry,patient_Recall@1_difference,17475,PRIME,0.0010302745500113918 +improve_benchmark,MHCflurry,patient_Recall@5_difference,17475,PRIME,-0.012298050222278895 +improve_benchmark,MHCflurry,patient_Recall@10_difference,17475,PRIME,-0.052622455850477076 +improve_benchmark,MHCflurry,patient_Recall@20_difference,17475,PRIME,-0.05794059326180068 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,2315,DeepHLApan,-0.03852733305541504 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,2315,DeepHLApan,0.05223555070883315 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,2315,DeepHLApan,0.08995819701926577 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,2315,DeepHLApan,-0.023209741912031956 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,2315,DeepHLApan,0.0 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,1015,DeepImmuno-CNN,0.10932207075221095 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,1015,DeepImmuno-CNN,0.02083333333333337 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,1015,DeepImmuno-CNN,0.029166666666666563 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,2315,MHCflurry,0.44386743408029294 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,2315,MHCflurry,0.06068702290076339 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,2315,MHCflurry,0.04249363867684475 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,2315,MHCflurry,-0.008651399491094058 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,2315,MHCflurry,0.0 +zhao_vaccine_benchmark,BigMHC,record_score_spearman,2310,PRIME,0.3699749982360709 +zhao_vaccine_benchmark,BigMHC,patient_Recall@1_difference,2310,PRIME,0.05258397932816539 +zhao_vaccine_benchmark,BigMHC,patient_Recall@5_difference,2310,PRIME,0.04298018949181737 +zhao_vaccine_benchmark,BigMHC,patient_Recall@10_difference,2310,PRIME,-0.015503875968992165 +zhao_vaccine_benchmark,BigMHC,patient_Recall@20_difference,2310,PRIME,0.0 +zhao_vaccine_benchmark,DeepHLApan,record_score_spearman,1015,DeepImmuno-CNN,0.40610422466771756 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,1015,DeepImmuno-CNN,-0.02083333333333326 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,1015,DeepImmuno-CNN,0.012499999999999956 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,1015,DeepImmuno-CNN,0.0 +zhao_vaccine_benchmark,DeepHLApan,record_score_spearman,2315,MHCflurry,-0.040780884614182465 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,2315,MHCflurry,0.008451472191930237 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,2315,MHCflurry,-0.047464558342421026 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,2315,MHCflurry,0.014558342420937898 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,2315,MHCflurry,0.0 +zhao_vaccine_benchmark,DeepHLApan,record_score_spearman,2310,PRIME,-0.14011603954761742 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,2310,PRIME,0.007290513104466573 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,2310,PRIME,-0.040620770271933226 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,2310,PRIME,0.008065706902916303 +zhao_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,2310,PRIME,0.0 +zhao_vaccine_benchmark,DeepImmuno-CNN,record_score_spearman,1015,MHCflurry,0.09367228092520263 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@1_difference,1015,MHCflurry,0.019791666666666652 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@5_difference,1015,MHCflurry,-0.007291666666666585 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@10_difference,1015,MHCflurry,-0.012499999999999956 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@20_difference,1015,MHCflurry,0.0 +zhao_vaccine_benchmark,DeepImmuno-CNN,record_score_spearman,1015,PRIME,-0.030588882476202253 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@1_difference,1015,PRIME,0.006249999999999978 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@5_difference,1015,PRIME,-0.005208333333333148 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@10_difference,1015,PRIME,-0.012499999999999956 +zhao_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@20_difference,1015,PRIME,0.0 +zhao_vaccine_benchmark,MHCflurry,record_score_spearman,2310,PRIME,0.7408911584698367 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@1_difference,2310,PRIME,-0.009043927648578837 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@5_difference,2310,PRIME,0.0075796726959517224 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@10_difference,2310,PRIME,-0.006718346253229868 +zhao_vaccine_benchmark,MHCflurry,patient_Recall@20_difference,2310,PRIME,0.0 +rcc_vaccine_benchmark,BigMHC,record_score_spearman,128,DeepHLApan,0.24047728053107473 +rcc_vaccine_benchmark,BigMHC,patient_Recall@1_difference,128,DeepHLApan,-0.010101010101010097 +rcc_vaccine_benchmark,BigMHC,patient_Recall@5_difference,128,DeepHLApan,-0.003575436908770213 +rcc_vaccine_benchmark,BigMHC,patient_Recall@10_difference,128,DeepHLApan,-0.010582010582010581 +rcc_vaccine_benchmark,BigMHC,patient_Recall@20_difference,128,DeepHLApan,0.0 +rcc_vaccine_benchmark,BigMHC,record_score_spearman,51,DeepImmuno-CNN,0.3616192718710379 +rcc_vaccine_benchmark,BigMHC,patient_Recall@1_difference,51,DeepImmuno-CNN,-0.028571428571428567 +rcc_vaccine_benchmark,BigMHC,patient_Recall@5_difference,51,DeepImmuno-CNN,0.0 +rcc_vaccine_benchmark,BigMHC,patient_Recall@10_difference,51,DeepImmuno-CNN,0.023809523809523725 +rcc_vaccine_benchmark,BigMHC,patient_Recall@20_difference,51,DeepImmuno-CNN,0.0 +rcc_vaccine_benchmark,BigMHC,record_score_spearman,128,PRIME,0.43052596790413744 +rcc_vaccine_benchmark,BigMHC,patient_Recall@1_difference,128,PRIME,-0.005691839025172354 +rcc_vaccine_benchmark,BigMHC,patient_Recall@5_difference,128,PRIME,-0.12574955908289237 +rcc_vaccine_benchmark,BigMHC,patient_Recall@10_difference,128,PRIME,-0.004489337822671136 +rcc_vaccine_benchmark,BigMHC,patient_Recall@20_difference,128,PRIME,0.0 +rcc_vaccine_benchmark,DeepHLApan,record_score_spearman,51,DeepImmuno-CNN,0.6475276218076436 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,51,DeepImmuno-CNN,-0.004761904761904759 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,51,DeepImmuno-CNN,1.1102230246251565e-16 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,51,DeepImmuno-CNN,0.04761904761904756 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,51,DeepImmuno-CNN,0.0 +rcc_vaccine_benchmark,DeepHLApan,record_score_spearman,128,PRIME,0.0359305767577579 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@1_difference,128,PRIME,0.004409171075837742 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@5_difference,128,PRIME,-0.12217412217412216 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@10_difference,128,PRIME,0.006092672759339446 +rcc_vaccine_benchmark,DeepHLApan,patient_Recall@20_difference,128,PRIME,0.0 +rcc_vaccine_benchmark,DeepImmuno-CNN,record_score_spearman,51,PRIME,0.2156361084619998 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@1_difference,51,PRIME,0.028571428571428567 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@5_difference,51,PRIME,-0.03571428571428581 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@10_difference,51,PRIME,-0.04761904761904756 +rcc_vaccine_benchmark,DeepImmuno-CNN,patient_Recall@20_difference,51,PRIME,0.0 diff --git a/results/analysis/stability/sensitivity_summary.csv b/results/analysis/stability/sensitivity_summary.csv new file mode 100644 index 0000000..4135855 --- /dev/null +++ b/results/analysis/stability/sensitivity_summary.csv @@ -0,0 +1,13 @@ +comparison,dataset,k,metric,predictors_in_support,strategy,value +BigMHC_vs_PRIME,improve_benchmark,5,probability_BigMHC_below_PRIME,,common_support,0.9185 +BigMHC_vs_PRIME,zhao_vaccine_benchmark,5,probability_BigMHC_below_PRIME,,common_support,0.0555 +BigMHC_vs_PRIME,rcc_vaccine_benchmark,5,probability_BigMHC_below_PRIME,,common_support,0.9985 +coverage_filtered_predictors,improve_benchmark,5,common_support_n,BigMHC|MHCflurry|PRIME,coverage_threshold_0.5,17475 +coverage_filtered_predictors,zhao_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|MHCflurry|PRIME,coverage_threshold_0.5,2310 +coverage_filtered_predictors,rcc_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|PRIME,coverage_threshold_0.5,128 +coverage_filtered_predictors,improve_benchmark,5,common_support_n,BigMHC|MHCflurry|PRIME,coverage_threshold_0.8,17475 +coverage_filtered_predictors,zhao_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|MHCflurry|PRIME,coverage_threshold_0.8,2310 +coverage_filtered_predictors,rcc_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|PRIME,coverage_threshold_0.8,128 +coverage_filtered_predictors,improve_benchmark,5,common_support_n,BigMHC|MHCflurry|PRIME,coverage_threshold_0.95,17475 +coverage_filtered_predictors,zhao_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|MHCflurry|PRIME,coverage_threshold_0.95,2310 +coverage_filtered_predictors,rcc_vaccine_benchmark,5,common_support_n,BigMHC|DeepHLApan|PRIME,coverage_threshold_0.95,128 diff --git a/results/manifest.json b/results/manifest.json index 1565194..852f8a1 100644 --- a/results/manifest.json +++ b/results/manifest.json @@ -156,9 +156,9 @@ "sha256": "1b2e9de3c7f7dfabc9c3241b7552db3cd8f7298e866a8f08ed2fe00a000b7635" }, { - "bytes": 487, + "bytes": 507, "path": "reports/manuscript_resource_audit.json", - "sha256": "90836e1d37e0eeaab46a0ab3781bc8811877c4c514570015d1546875fd3cdcd0" + "sha256": "f3ecb57f6e562c5737aa5755fe2a42a0f2cafb304aa18b9e45c7f7c6d4e39a10" }, { "bytes": 11059, @@ -176,9 +176,9 @@ "sha256": "54dc400f5e5970188b4822ae9253e17c3e2b18a463056a2e629f7f884e44589d" }, { - "bytes": 889, + "bytes": 1314, "path": "reports/clean_reproduction.json", - "sha256": "7ff012366417a31a5136f85e7a3be6bcd9135a77545078d26938abb699b3c02d" + "sha256": "a5ec65a24bafd59a03d383d261c45f95cfaeea74c69770603307435aab96897b" }, { "bytes": 3957, @@ -216,14 +216,14 @@ "sha256": "537cb1397a68c114f4e49db1dd07be57fcab435bc2849d342f55399b650ac18e" }, { - "bytes": 28762, + "bytes": 32388, "path": "paper/manuscript_resource.md", - "sha256": "14e5b9707bc9e674ae9786d9cfee76af78a8228a1952838e213d903b1f8bbed4" + "sha256": "478a604932072c91b847672417a325ba34a7ca5f42072ae7900e16f1e119e347" }, { - "bytes": 4697, + "bytes": 5021, "path": "paper/references.bib", - "sha256": "5d883804d7d2be339135d8b3aa9efe2f4798ecadf4405a25b6318848c210216a" + "sha256": "272e420b33156882a5c53fed86b7c1eb22bdb7d29da916967fbc68f0f60266c1" }, { "bytes": 5320, @@ -236,9 +236,9 @@ "sha256": "3bc706c7b062876fcdec8c1fb12c788baacb2302ed4eb4beef67961ded4ac792" }, { - "bytes": 7407, + "bytes": 8573, "path": "FINAL_REPORT.md", - "sha256": "1caaadde6d784865bdd1ad2c659904f806f62d025e3b31c8e648f952c90ad6e7" + "sha256": "cbbb3e4b04a105c19d3ca3bce1cd3bf434a0a57ceaff4c1e25a7dd426f6523fc" }, { "bytes": 1239894, @@ -369,6 +369,181 @@ "bytes": 288, "path": "reports/extension_clean_reproduction.json", "sha256": "f60acbb3d208c6971bf129be0c5d7be39dd2784147918005ff794b2ead558fa7" + }, + { + "bytes": 49526, + "path": "data/processed/rcc_vaccine_benchmark.csv", + "sha256": "1697b5808a05836995fbc8e8167d3516187f4184367fa7e1c2564b4701f6b3f8" + }, + { + "bytes": 49526, + "path": "data/processed/rcc_vaccine_benchmark_full.csv", + "sha256": "1697b5808a05836995fbc8e8167d3516187f4184367fa7e1c2564b4701f6b3f8" + }, + { + "bytes": 288, + "path": "data/rcc_vaccine_summary.json", + "sha256": "ff829adc508c37ca50fe06d5ddf7505c51373f837dcd8f06770bebad85eb0977" + }, + { + "bytes": 2861, + "path": "research/extension_protocol_rcc_v1.json", + "sha256": "fa3e89478b91e03500f1ad65327fe1a803624ea5ae88e782a1b466932fd2b399" + }, + { + "bytes": 12340, + "path": "research/training_overlap_audit_rcc.csv", + "sha256": "4381cd1b1a2b3061a8e26bb4d7deec8ff4606eb115e6c2ade4d3787051cbdb53" + }, + { + "bytes": 1070, + "path": "research/training_overlap_summary_rcc.json", + "sha256": "f8315e6c6115fbbd01fdb276d9bdbb68f7d9911c6da89a77c6e037ac7da42721" + }, + { + "bytes": 13826, + "path": "results/raw_predictions/rcc/bigmhc-v1.0.csv", + "sha256": "1b16e7bf545032a9763a9c899190775dddf24ea19378d8763899663a460f8dd5" + }, + { + "bytes": 12737, + "path": "results/raw_predictions/rcc/prime-2.0.csv", + "sha256": "68ed9f4f7a928964ccbab018d315e0e3ff9315d1b5c5a80665c30dee9a861140" + }, + { + "bytes": 13004, + "path": "results/raw_predictions/rcc/deepimmuno-cnn.csv", + "sha256": "be8fdb1f4efff6ef3159971a46f58513e8cc90d2d2f5737494aa14871700ad1e" + }, + { + "bytes": 15155, + "path": "results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "sha256": "4d084a12522852fbdfe3bcbe9b64a8477f496b2073e6f8fc95943bfd18b3e4d4" + }, + { + "bytes": 5035, + "path": "reports/rcc_predictor_run.json", + "sha256": "c5efc343344eff9f5c962f8d7834c7f3c0c686bcc669eaaf87617b44f0da3b00" + }, + { + "bytes": 82938, + "path": "results/analysis/rcc/metrics.json", + "sha256": "6da6a361e1f7bf5596e88820cb2c2c00d549e3152579f30e217da446e2daf859" + }, + { + "bytes": 266, + "path": "results/analysis/rcc/missingness.csv", + "sha256": "75b876d69948ea61012702b39a0382bed0a4b41ed40890c4985a0827a14a1d33" + }, + { + "bytes": 11450, + "path": "results/analysis/rcc/paired_differences.csv", + "sha256": "cc67f00b4b278f0614bcac1b2b4ef9513ecc6abbb144bce1d44696e309885449" + }, + { + "bytes": 1402, + "path": "results/analysis/stability/analysis_metadata.json", + "sha256": "a09048edd10e12ae98583d304f2f21c960f222a42ed6ca46f4089c43c6b46e3e" + }, + { + "bytes": 4538, + "path": "results/analysis/stability/dataset_predictor_metric_matrix.csv", + "sha256": "f8828927ff5b5aac296ad562145d95bc2dc50768478c512771f8044a80b6c0bb" + }, + { + "bytes": 8609, + "path": "results/analysis/stability/rank_stability.csv", + "sha256": "5c8b4b111f938788827184399f3fd063033fde55c33f8d22c1f9683bdbba2101" + }, + { + "bytes": 760, + "path": "results/analysis/stability/model_selection_first_probability.csv", + "sha256": "0ab7bae116a8a227aa079b35bafc1a17bc691cd121547c75f2975d176f028c14" + }, + { + "bytes": 1463, + "path": "results/analysis/stability/sensitivity_summary.csv", + "sha256": "96ec210d87df0fae602a35da0e2a9cdcd7554ad7aa8de9059d685525de20a8a4" + }, + { + "bytes": 1913, + "path": "results/analysis/stability/leave_one_domain_out.csv", + "sha256": "d35e7c8f98211daa299761df8a9cafdbd68528fae5d50252c55a25017a7cac7b" + }, + { + "bytes": 375, + "path": "results/analysis/stability/endpoint_domain_metadata.csv", + "sha256": "bb381081a5abc3cf45265d73aa812675fd2d7eac8319560eee4e7f918d1f0170" + }, + { + "bytes": 3655, + "path": "results/analysis/stability/endpoint_domain_auroc.svg", + "sha256": "8f9875a1ff7b0e75be13cd4c31e7623a1d4d14bd45481a8b2dad85e2abffb9ab" + }, + { + "bytes": 91312, + "path": "results/analysis/stability/endpoint_domain_auroc.png", + "sha256": "327fcd4d215d48f18832cf77959e125fc2d2f64490a80bd86cf3c9dd316c956b" + }, + { + "bytes": 348, + "path": "contracts/dataset-card.example.json", + "sha256": "4c3abe7cd9c09d8560df1b7c2530461fec81645442917c50969b0bed379c9c67" + }, + { + "bytes": 360, + "path": "contracts/predictor-card.example.json", + "sha256": "f2f0c6e1490f622e45b4c1b2e04076f0836632cf6aff08c92c70044be045a7cd" + }, + { + "bytes": 575, + "path": "contracts/prediction-artifact.schema.json", + "sha256": "ed38da54c4fc933ca616d3d88c57e894b97c6dc079962e542f6d5eae5ba32cbe" + }, + { + "bytes": 153, + "path": "contracts/synthetic/benchmark.csv", + "sha256": "06696235cd66940fe2bc88b15a1fb37cc5d085331ff652a1e393fde53b9cb310" + }, + { + "bytes": 283, + "path": "contracts/synthetic/predictions.csv", + "sha256": "ed054ea5f4da890ad676049d297eddc0243375de392a59594655ea24c56db858" + }, + { + "bytes": 321, + "path": "predictors/mhcmatch/attempts/sweep_receipt.json", + "sha256": "98b8977360a1a9f1fe031edf02692a7bbdf4c966c87cd7060d9528843126d575" + }, + { + "bytes": 321, + "path": "predictors/mhcnuggets/attempts/sweep_receipt.json", + "sha256": "f330557e0a8b5c092b4ab06bc18062233ae9c832021ff7419fde61b5d95d34ff" + }, + { + "bytes": 322, + "path": "predictors/neofox/attempts/sweep_receipt.json", + "sha256": "820962cf5c57c19542e89f45a34135ef414c186a9b5cfcf58894b3a6c0e64073" + }, + { + "bytes": 325, + "path": "predictors/neoguider/attempts/sweep_receipt.json", + "sha256": "5fee464b36230c75e18e02655dfa7dba45cca11a8c7062baf33201fb86cef123" + }, + { + "bytes": 344, + "path": "predictors/pvactools/attempts/sweep_receipt.json", + "sha256": "fdd6e350bdd290a5bd6d92a0d4c88e12203e185d96910f7c5cf0d2764361b18d" + }, + { + "bytes": 315, + "path": "predictors/seq2neo/attempts/sweep_receipt.json", + "sha256": "9e65865499af2e52047fa59bbac63a68484a5ef4cd5e65b3b6dbe68671bf26c7" + }, + { + "bytes": 312, + "path": "predictors/vaxrank/attempts/sweep_receipt.json", + "sha256": "8cb0d6206ce785f351b09acff12d0676462744653a48856608bc183a773d9365" } ], "benchmark": { @@ -379,12 +554,12 @@ "studies": 3 }, "clean_reproduction": { - "command": "make -j4 reproduce-results", - "completed_at": "2026-08-20T12:29:54Z", - "elapsed_seconds": 734, + "command": "make -C /tmp/neorepro-release-hcLY3g -j4 reproduce-results", + "completed_at": "2026-08-21T07:53:30Z", "expected_manifest_change": { - "project_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "project_worktree_clean": true + "project_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", + "project_worktree_clean": false, + "reason": "the manifest is generated before the reproduction's final test step" }, "initial_worktree_clean": true, "returncode": 0, @@ -396,14 +571,18 @@ "scipy": "1.17.1" }, "schema_version": "1.0", - "source_commit": "f4d86e6fbffb8062952eab3ec0d6d236a936a67d", - "started_at": "2026-08-20T12:17:40Z", + "source_commit": "5ff644dd210f7f672ec5ad25b4e64a0c5c6e28f1", "verification": { - "byte_stable_tracked_artifacts": true, - "maximum_metric_absolute_error": 1.1102230246251565e-16, - "metric_validation": "24 checks passed at tolerance 1e-12", - "post_run_changed_files_excluding_manifest": [], - "pytest": "27 passed", + "generated_outputs_promoted_from_clean_clone": true, + "manuscript_audit": "pass; 12 citations resolved; no placeholders or forbidden overclaims", + "maximum_metric_absolute_error": 2.220446049250313e-16, + "metric_validation": "34 checks passed at tolerance 1e-12", + "post_run_findings": [ + "legacy generated CSV files used CRLF and were normalized to the writer's declared LF output", + "RCC metrics contained invocation-dependent absolute paths and were replaced by clean-clone relative paths" + ], + "pytest": "49 passed", + "release_boundary": "pass; 413 tracked files; zero violations", "ruff": "pass" } }, @@ -434,7 +613,7 @@ "MHCflurry": "2.2.1", "PRIME": "2.0" }, - "project_commit": "9fa3acb763c8c22e94653fa54162d24c3a55bbe0", + "project_commit": "d8710676d758a51701121c6ab04c32183c8ac553", "project_worktree_clean": false, "schema_version": "1.0" } diff --git a/results/raw_predictions/rcc/bigmhc-v1.0.csv b/results/raw_predictions/rcc/bigmhc-v1.0.csv new file mode 100644 index 0000000..bbfbfa0 --- /dev/null +++ b/results/raw_predictions/rcc/bigmhc-v1.0.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,bigmhc_im,bigmhc_el +rcc-vaccine-8c2e1ec6d8f675cf,BigMHC,v1.0,immunogenicity,0.8492275,higher,predicted,0.8492275,0.998158 +rcc-vaccine-6a6ffb8b81ec9208,BigMHC,v1.0,immunogenicity,0.58425915,higher,predicted,0.58425915,0.16537319 +rcc-vaccine-4adeedbe0e35cf09,BigMHC,v1.0,immunogenicity,,higher,unsupported_hla,, +rcc-vaccine-e06799bdf8de8af5,BigMHC,v1.0,immunogenicity,0.8492275,higher,predicted,0.8492275,0.998158 +rcc-vaccine-969998ab9d8b4dc1,BigMHC,v1.0,immunogenicity,0.00042694155,higher,predicted,0.00042694155,0.0001798583 +rcc-vaccine-a5952e9e71bd27bd,BigMHC,v1.0,immunogenicity,0.19695829,higher,predicted,0.19695829,0.15335162 +rcc-vaccine-6fa3e9ad450e030e,BigMHC,v1.0,immunogenicity,0.8492275,higher,predicted,0.8492275,0.998158 +rcc-vaccine-8a9041cc55b08e38,BigMHC,v1.0,immunogenicity,0.0199301,higher,predicted,0.0199301,0.026195873 +rcc-vaccine-e11c81feedc5751e,BigMHC,v1.0,immunogenicity,0.03303969,higher,predicted,0.03303969,0.003229972 +rcc-vaccine-fefffdcccca8649c,BigMHC,v1.0,immunogenicity,0.2625458,higher,predicted,0.2625458,0.5432058 +rcc-vaccine-940b62df9c3078c4,BigMHC,v1.0,immunogenicity,0.051157277,higher,predicted,0.051157277,0.3431107 +rcc-vaccine-43742d14e082c2e6,BigMHC,v1.0,immunogenicity,0.00051022216,higher,predicted,0.00051022216,1.4941308e-05 +rcc-vaccine-769c6b305827c570,BigMHC,v1.0,immunogenicity,0.17434175,higher,predicted,0.17434175,0.014034523 +rcc-vaccine-e74f183589095f61,BigMHC,v1.0,immunogenicity,0.14538826,higher,predicted,0.14538826,0.0007323162 +rcc-vaccine-15ab70cb934aad8f,BigMHC,v1.0,immunogenicity,0.1274003,higher,predicted,0.1274003,0.0017386841 +rcc-vaccine-c8a8ce5ab15f9d4b,BigMHC,v1.0,immunogenicity,0.07331483,higher,predicted,0.07331483,0.0027492247 +rcc-vaccine-f05602883865f4bf,BigMHC,v1.0,immunogenicity,0.094515726,higher,predicted,0.094515726,0.41674283 +rcc-vaccine-b92105a12fb6f028,BigMHC,v1.0,immunogenicity,0.1498395,higher,predicted,0.1498395,0.5488981 +rcc-vaccine-a004075aed453bcf,BigMHC,v1.0,immunogenicity,0.19785257,higher,predicted,0.19785257,0.007729145 +rcc-vaccine-70eb710f90e2973b,BigMHC,v1.0,immunogenicity,0.037451383,higher,predicted,0.037451383,0.05899397 +rcc-vaccine-c81528933be4a147,BigMHC,v1.0,immunogenicity,0.032642193,higher,predicted,0.032642193,0.7174736 +rcc-vaccine-2a2f0b8f959aa3f6,BigMHC,v1.0,immunogenicity,0.22346842,higher,predicted,0.22346842,0.07276444 +rcc-vaccine-74e8a99b7ef86c61,BigMHC,v1.0,immunogenicity,0.6325035,higher,predicted,0.6325035,0.9122117 +rcc-vaccine-88bea2527dd72a1d,BigMHC,v1.0,immunogenicity,0.11432793,higher,predicted,0.11432793,0.012796485 +rcc-vaccine-ff6bbb8d69468a47,BigMHC,v1.0,immunogenicity,0.07834623,higher,predicted,0.07834623,0.12306678 +rcc-vaccine-8ea6c4f21d2c81ee,BigMHC,v1.0,immunogenicity,0.0004803969,higher,predicted,0.0004803969,0.00025231813 +rcc-vaccine-b2f6110d808ac0da,BigMHC,v1.0,immunogenicity,0.12104134,higher,predicted,0.12104134,0.04732251 +rcc-vaccine-f43e93564087f160,BigMHC,v1.0,immunogenicity,0.112379804,higher,predicted,0.112379804,0.6370787 +rcc-vaccine-fc1276f53c3f45e1,BigMHC,v1.0,immunogenicity,0.013032401,higher,predicted,0.013032401,0.00011015074 +rcc-vaccine-3e6b1c1d8de0ce71,BigMHC,v1.0,immunogenicity,0.10996994,higher,predicted,0.10996994,0.0041871676 +rcc-vaccine-4255bdded9e984bd,BigMHC,v1.0,immunogenicity,0.20445038,higher,predicted,0.20445038,0.3118517 +rcc-vaccine-31c3a5a1c962db43,BigMHC,v1.0,immunogenicity,0.33050114,higher,predicted,0.33050114,0.98251694 +rcc-vaccine-d3d4046d155047b6,BigMHC,v1.0,immunogenicity,0.4889801,higher,predicted,0.4889801,0.6427992 +rcc-vaccine-5e4096b57e93c92e,BigMHC,v1.0,immunogenicity,0.013662019,higher,predicted,0.013662019,0.048441563 +rcc-vaccine-e8d6aeb51705642a,BigMHC,v1.0,immunogenicity,0.035264768,higher,predicted,0.035264768,0.3544042 +rcc-vaccine-26dc24e186221fe2,BigMHC,v1.0,immunogenicity,0.120982565,higher,predicted,0.120982565,0.35466275 +rcc-vaccine-364822c022710958,BigMHC,v1.0,immunogenicity,0.6495637,higher,predicted,0.6495637,0.4203276 +rcc-vaccine-9c477d13a3359f5e,BigMHC,v1.0,immunogenicity,0.2634864,higher,predicted,0.2634864,0.0073506013 +rcc-vaccine-487a8e43d12c17ac,BigMHC,v1.0,immunogenicity,0.64040625,higher,predicted,0.64040625,0.69878644 +rcc-vaccine-539b50b223ea0ee1,BigMHC,v1.0,immunogenicity,0.15084496,higher,predicted,0.15084496,0.95499474 +rcc-vaccine-618d6aa7d8232657,BigMHC,v1.0,immunogenicity,0.1541317,higher,predicted,0.1541317,0.0509347 +rcc-vaccine-71f5d2ab19a389e8,BigMHC,v1.0,immunogenicity,0.012482004,higher,predicted,0.012482004,0.16006315 +rcc-vaccine-409a0661a319bbb1,BigMHC,v1.0,immunogenicity,0.2634864,higher,predicted,0.2634864,0.0073506013 +rcc-vaccine-9f431d040990f0f7,BigMHC,v1.0,immunogenicity,0.018202517,higher,predicted,0.018202517,0.63248533 +rcc-vaccine-7c1009a0bb0524b3,BigMHC,v1.0,immunogenicity,0.02231897,higher,predicted,0.02231897,0.0074036336 +rcc-vaccine-42e869aba90dc4bd,BigMHC,v1.0,immunogenicity,0.00010520963,higher,predicted,0.00010520963,8.958782e-05 +rcc-vaccine-a1a88d1c11b4fcdd,BigMHC,v1.0,immunogenicity,0.04779095,higher,predicted,0.04779095,0.0033836025 +rcc-vaccine-d42b9fb3a44366ac,BigMHC,v1.0,immunogenicity,0.15084496,higher,predicted,0.15084496,0.95499474 +rcc-vaccine-ce806d8b1a4132b2,BigMHC,v1.0,immunogenicity,0.1408995,higher,predicted,0.1408995,0.8896032 +rcc-vaccine-ea51c18cc4e16995,BigMHC,v1.0,immunogenicity,0.1541317,higher,predicted,0.1541317,0.0509347 +rcc-vaccine-9bce007ba3a4aff0,BigMHC,v1.0,immunogenicity,0.010021402,higher,predicted,0.010021402,0.0002278478 +rcc-vaccine-aa3019acbdddb7dc,BigMHC,v1.0,immunogenicity,0.020995839,higher,predicted,0.020995839,0.00015733231 +rcc-vaccine-1bcdaed331509a08,BigMHC,v1.0,immunogenicity,0.1408995,higher,predicted,0.1408995,0.8896032 +rcc-vaccine-4a1853322eea7a14,BigMHC,v1.0,immunogenicity,0.008962676,higher,predicted,0.008962676,0.00016499302 +rcc-vaccine-cde49959d75e6ec2,BigMHC,v1.0,immunogenicity,0.034591164,higher,predicted,0.034591164,0.05231158 +rcc-vaccine-7f8e1f8e02332663,BigMHC,v1.0,immunogenicity,0.028659904,higher,predicted,0.028659904,0.8122644 +rcc-vaccine-b8aa4d97d214f765,BigMHC,v1.0,immunogenicity,0.04871258,higher,predicted,0.04871258,0.0059868065 +rcc-vaccine-5993797f7a6c7057,BigMHC,v1.0,immunogenicity,0.1504514,higher,predicted,0.1504514,0.078459814 +rcc-vaccine-eacb2f310488ca9d,BigMHC,v1.0,immunogenicity,0.25954965,higher,predicted,0.25954965,0.43180725 +rcc-vaccine-70f05fcfa038224f,BigMHC,v1.0,immunogenicity,0.0045895926,higher,predicted,0.0045895926,0.06761794 +rcc-vaccine-4d15c4e62bb19311,BigMHC,v1.0,immunogenicity,0.0026878978,higher,predicted,0.0026878978,0.0007637069 +rcc-vaccine-a6b2180415ff1b56,BigMHC,v1.0,immunogenicity,0.02717142,higher,predicted,0.02717142,0.17152806 +rcc-vaccine-e52f3609d694b4ba,BigMHC,v1.0,immunogenicity,0.15067343,higher,predicted,0.15067343,0.01708421 +rcc-vaccine-763f29ff7ac4c65d,BigMHC,v1.0,immunogenicity,0.10464774,higher,predicted,0.10464774,0.017464515 +rcc-vaccine-c331f65899396b9b,BigMHC,v1.0,immunogenicity,0.05073751,higher,predicted,0.05073751,0.16568084 +rcc-vaccine-09c9b699d977f421,BigMHC,v1.0,immunogenicity,0.056319688,higher,predicted,0.056319688,0.6093203 +rcc-vaccine-e944d5a782a7d074,BigMHC,v1.0,immunogenicity,0.40012008,higher,predicted,0.40012008,0.039899163 +rcc-vaccine-75f86cf28dca60e2,BigMHC,v1.0,immunogenicity,0.076639615,higher,predicted,0.076639615,0.118811674 +rcc-vaccine-5b2ef450eaf2248a,BigMHC,v1.0,immunogenicity,0.21340606,higher,predicted,0.21340606,0.88489115 +rcc-vaccine-c50e7f0e1b3b69a5,BigMHC,v1.0,immunogenicity,0.22124277,higher,predicted,0.22124277,0.5948638 +rcc-vaccine-ae70f68909b113e8,BigMHC,v1.0,immunogenicity,0.10368972,higher,predicted,0.10368972,0.05833147 +rcc-vaccine-efdf21635c142ab8,BigMHC,v1.0,immunogenicity,0.012091301,higher,predicted,0.012091301,0.013448933 +rcc-vaccine-817b1ac6328997a7,BigMHC,v1.0,immunogenicity,0.0039570313,higher,predicted,0.0039570313,0.03172536 +rcc-vaccine-feb66d3c66c8a4de,BigMHC,v1.0,immunogenicity,0.14428535,higher,predicted,0.14428535,0.55323803 +rcc-vaccine-7946e9a10fe114cf,BigMHC,v1.0,immunogenicity,0.009083572,higher,predicted,0.009083572,0.007149222 +rcc-vaccine-f3e32e857dd09d71,BigMHC,v1.0,immunogenicity,0.098906696,higher,predicted,0.098906696,0.42554325 +rcc-vaccine-240bf5b232815c6a,BigMHC,v1.0,immunogenicity,0.35946846,higher,predicted,0.35946846,0.5475027 +rcc-vaccine-40c01a17a3ed4a2e,BigMHC,v1.0,immunogenicity,0.04898708,higher,predicted,0.04898708,0.11417298 +rcc-vaccine-7b823cc728ea1450,BigMHC,v1.0,immunogenicity,0.08569368,higher,predicted,0.08569368,0.9577692 +rcc-vaccine-0c7eb5e3295f0ad2,BigMHC,v1.0,immunogenicity,0.012339222,higher,predicted,0.012339222,0.09743548 +rcc-vaccine-88aaf2311ce63196,BigMHC,v1.0,immunogenicity,0.14953732,higher,predicted,0.14953732,0.11653518 +rcc-vaccine-bcae618cfad77c1f,BigMHC,v1.0,immunogenicity,0.019738207,higher,predicted,0.019738207,0.086946145 +rcc-vaccine-11f61d47947266cc,BigMHC,v1.0,immunogenicity,0.046728577,higher,predicted,0.046728577,0.43281174 +rcc-vaccine-f4a4db2362e7a54e,BigMHC,v1.0,immunogenicity,0.58805186,higher,predicted,0.58805186,0.7872415 +rcc-vaccine-1eceb5b6a969d721,BigMHC,v1.0,immunogenicity,0.64963526,higher,predicted,0.64963526,0.73666704 +rcc-vaccine-55106745d24cfbf4,BigMHC,v1.0,immunogenicity,0.15317993,higher,predicted,0.15317993,0.2744644 +rcc-vaccine-a808947689fbe42c,BigMHC,v1.0,immunogenicity,0.037526943,higher,predicted,0.037526943,0.4084019 +rcc-vaccine-590f5ea08f24e2cf,BigMHC,v1.0,immunogenicity,0.11108919,higher,predicted,0.11108919,0.03396273 +rcc-vaccine-b984dc3be5c1fa17,BigMHC,v1.0,immunogenicity,0.58805186,higher,predicted,0.58805186,0.7872415 +rcc-vaccine-6ff6e758ac279425,BigMHC,v1.0,immunogenicity,0.20403412,higher,predicted,0.20403412,0.030309463 +rcc-vaccine-aa0d4e0503dacba4,BigMHC,v1.0,immunogenicity,0.096679814,higher,predicted,0.096679814,0.043357793 +rcc-vaccine-4d44829f31cf5558,BigMHC,v1.0,immunogenicity,0.09515708,higher,predicted,0.09515708,0.33479306 +rcc-vaccine-4f7d9f9f22d48440,BigMHC,v1.0,immunogenicity,0.31902334,higher,predicted,0.31902334,0.22489502 +rcc-vaccine-60d1ba1b8ae350fd,BigMHC,v1.0,immunogenicity,0.4844474,higher,predicted,0.4844474,0.8657421 +rcc-vaccine-7ec85836cb09ee30,BigMHC,v1.0,immunogenicity,0.008745876,higher,predicted,0.008745876,0.08516661 +rcc-vaccine-4b7fdcc54f65b98b,BigMHC,v1.0,immunogenicity,0.4844474,higher,predicted,0.4844474,0.8657421 +rcc-vaccine-d02de725ecc685a5,BigMHC,v1.0,immunogenicity,0.3566296,higher,predicted,0.3566296,0.11564131 +rcc-vaccine-e1d984ebf245728f,BigMHC,v1.0,immunogenicity,0.09470768,higher,predicted,0.09470768,0.08543001 +rcc-vaccine-a9a3c2bc98a74746,BigMHC,v1.0,immunogenicity,0.03360454,higher,predicted,0.03360454,0.64224684 +rcc-vaccine-da610226e338a8de,BigMHC,v1.0,immunogenicity,0.48562998,higher,predicted,0.48562998,0.72100705 +rcc-vaccine-ca0a8a8bb4e2d094,BigMHC,v1.0,immunogenicity,0.07511683,higher,predicted,0.07511683,0.0056161615 +rcc-vaccine-c45585f253341070,BigMHC,v1.0,immunogenicity,0.38412148,higher,predicted,0.38412148,0.8916814 +rcc-vaccine-b2761351ccb757b7,BigMHC,v1.0,immunogenicity,0.03360454,higher,predicted,0.03360454,0.64224684 +rcc-vaccine-b1ad936ccc2f249a,BigMHC,v1.0,immunogenicity,0.38412148,higher,predicted,0.38412148,0.8916814 +rcc-vaccine-02261377bb800592,BigMHC,v1.0,immunogenicity,0.009326143,higher,predicted,0.009326143,0.20740989 +rcc-vaccine-834aff6d2a7dc363,BigMHC,v1.0,immunogenicity,0.06686703,higher,predicted,0.06686703,0.07113467 +rcc-vaccine-0036b8c45a4cd20d,BigMHC,v1.0,immunogenicity,0.03360454,higher,predicted,0.03360454,0.64224684 +rcc-vaccine-08447808dd0300dd,BigMHC,v1.0,immunogenicity,0.38412148,higher,predicted,0.38412148,0.8916814 +rcc-vaccine-34c3e7e7a2dcd8b6,BigMHC,v1.0,immunogenicity,0.009326143,higher,predicted,0.009326143,0.20740989 +rcc-vaccine-1ae968c5ae01f49b,BigMHC,v1.0,immunogenicity,0.06686703,higher,predicted,0.06686703,0.07113467 +rcc-vaccine-29f7c92d094c1ced,BigMHC,v1.0,immunogenicity,0.040688813,higher,predicted,0.040688813,0.55616236 +rcc-vaccine-0aaef5904890371c,BigMHC,v1.0,immunogenicity,0.06686703,higher,predicted,0.06686703,0.07113467 +rcc-vaccine-27b47d43dab2b353,BigMHC,v1.0,immunogenicity,0.69446176,higher,predicted,0.69446176,0.30978775 +rcc-vaccine-46b486b9d199e93a,BigMHC,v1.0,immunogenicity,0.13332154,higher,predicted,0.13332154,0.026609246 +rcc-vaccine-b8a9c44bcf53568f,BigMHC,v1.0,immunogenicity,0.30859342,higher,predicted,0.30859342,0.4905243 +rcc-vaccine-7425139a6230adc0,BigMHC,v1.0,immunogenicity,0.35320142,higher,predicted,0.35320142,0.6920496 +rcc-vaccine-e39c49c350f42009,BigMHC,v1.0,immunogenicity,0.14485602,higher,predicted,0.14485602,0.90685815 +rcc-vaccine-73717d4619b66faf,BigMHC,v1.0,immunogenicity,0.3526861,higher,predicted,0.3526861,0.08524216 +rcc-vaccine-f130cbb5ea1ca0fa,BigMHC,v1.0,immunogenicity,0.003947547,higher,predicted,0.003947547,0.0018933918 +rcc-vaccine-3f16099c911d2791,BigMHC,v1.0,immunogenicity,0.35320142,higher,predicted,0.35320142,0.6920496 +rcc-vaccine-07bc8054e1edd65a,BigMHC,v1.0,immunogenicity,0.41273668,higher,predicted,0.41273668,0.94675225 +rcc-vaccine-99bd603a759d6b22,BigMHC,v1.0,immunogenicity,0.50416905,higher,predicted,0.50416905,0.8801918 +rcc-vaccine-ad726d8ef91bd0e6,BigMHC,v1.0,immunogenicity,0.243946,higher,predicted,0.243946,0.0207863 +rcc-vaccine-972e1f28fe4cb96a,BigMHC,v1.0,immunogenicity,0.0084418915,higher,predicted,0.0084418915,0.0074836127 +rcc-vaccine-e0d628668c1eccf2,BigMHC,v1.0,immunogenicity,0.007305874,higher,predicted,0.007305874,5.319204e-05 +rcc-vaccine-3d165d772065bb3a,BigMHC,v1.0,immunogenicity,0.015850548,higher,predicted,0.015850548,0.61480916 +rcc-vaccine-9ef49682993d2ea1,BigMHC,v1.0,immunogenicity,0.024873625,higher,predicted,0.024873625,0.0945738 +rcc-vaccine-6c01a27a6a575007,BigMHC,v1.0,immunogenicity,0.21619721,higher,predicted,0.21619721,0.08226664 +rcc-vaccine-12b152d35907e280,BigMHC,v1.0,immunogenicity,0.09390874,higher,predicted,0.09390874,0.6517529 diff --git a/results/raw_predictions/rcc/deephlapan-1.1.1.csv b/results/raw_predictions/rcc/deephlapan-1.1.1.csv new file mode 100644 index 0000000..862d834 --- /dev/null +++ b/results/raw_predictions/rcc/deephlapan-1.1.1.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,binding_score +rcc-vaccine-8c2e1ec6d8f675cf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.956750869751,higher,predicted,0.999957203865 +rcc-vaccine-6a6ffb8b81ec9208,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0342342741787,higher,predicted,0.995593428612 +rcc-vaccine-4adeedbe0e35cf09,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,,higher,unsupported_hla, +rcc-vaccine-e06799bdf8de8af5,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.956750869751,higher,predicted,0.999957203865 +rcc-vaccine-969998ab9d8b4dc1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.10936331749,higher,predicted,0.969243168831 +rcc-vaccine-a5952e9e71bd27bd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.672411322594,higher,predicted,0.970010876656 +rcc-vaccine-6fa3e9ad450e030e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.956750869751,higher,predicted,0.999957203865 +rcc-vaccine-8a9041cc55b08e38,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.415020555258,higher,predicted,0.823912143707 +rcc-vaccine-e11c81feedc5751e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.271832853556,higher,predicted,0.853784382343 +rcc-vaccine-fefffdcccca8649c,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.663122951984,higher,predicted,0.990222096443 +rcc-vaccine-940b62df9c3078c4,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.288684308529,higher,predicted,0.945045113564 +rcc-vaccine-43742d14e082c2e6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0819523632526,higher,predicted,0.478505432606 +rcc-vaccine-769c6b305827c570,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.949861049652,higher,predicted,0.997111976147 +rcc-vaccine-e74f183589095f61,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.44372677803,higher,predicted,0.493312120438 +rcc-vaccine-15ab70cb934aad8f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.708318173885,higher,predicted,0.989219069481 +rcc-vaccine-c8a8ce5ab15f9d4b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.773888707161,higher,predicted,0.0873938798904 +rcc-vaccine-f05602883865f4bf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.954157352448,higher,predicted,0.991816818714 +rcc-vaccine-b92105a12fb6f028,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.929277539253,higher,predicted,0.99683535099 +rcc-vaccine-a004075aed453bcf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.773697257042,higher,predicted,0.662531495094 +rcc-vaccine-70eb710f90e2973b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.638193964958,higher,predicted,0.979091465473 +rcc-vaccine-c81528933be4a147,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0489731431007,higher,predicted,0.960151314735 +rcc-vaccine-2a2f0b8f959aa3f6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.54920065403,higher,predicted,0.952247798443 +rcc-vaccine-74e8a99b7ef86c61,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.364848554134,higher,predicted,0.999667048454 +rcc-vaccine-88bea2527dd72a1d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.996774017811,higher,predicted,0.935574650764 +rcc-vaccine-ff6bbb8d69468a47,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.991322398186,higher,predicted,0.856530070305 +rcc-vaccine-8ea6c4f21d2c81ee,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.904866814613,higher,predicted,0.830239772797 +rcc-vaccine-b2f6110d808ac0da,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.900792896748,higher,predicted,0.475547015667 +rcc-vaccine-f43e93564087f160,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.981161594391,higher,predicted,0.982510209084 +rcc-vaccine-fc1276f53c3f45e1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.781159996986,higher,predicted,0.835298418999 +rcc-vaccine-3e6b1c1d8de0ce71,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.991661071777,higher,predicted,0.868157505989 +rcc-vaccine-4255bdded9e984bd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.797833561897,higher,predicted,0.99191236496 +rcc-vaccine-31c3a5a1c962db43,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.282823115587,higher,predicted,0.99037283659 +rcc-vaccine-d3d4046d155047b6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.980403542519,higher,predicted,0.202609583735 +rcc-vaccine-5e4096b57e93c92e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0258417315781,higher,predicted,0.151212573051 +rcc-vaccine-e8d6aeb51705642a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.995793521404,higher,predicted,0.207422688603 +rcc-vaccine-26dc24e186221fe2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.730694293976,higher,predicted,0.968062222004 +rcc-vaccine-364822c022710958,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.13982026279,higher,predicted,0.999332427979 +rcc-vaccine-9c477d13a3359f5e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.691724777222,higher,predicted,0.993195176125 +rcc-vaccine-487a8e43d12c17ac,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.668253481388,higher,predicted,0.998432040215 +rcc-vaccine-539b50b223ea0ee1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.418166160583,higher,predicted,0.998265862465 +rcc-vaccine-618d6aa7d8232657,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.731171905994,higher,predicted,0.997755050659 +rcc-vaccine-71f5d2ab19a389e8,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0762045830488,higher,predicted,0.461806923151 +rcc-vaccine-409a0661a319bbb1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.691724777222,higher,predicted,0.993195176125 +rcc-vaccine-9f431d040990f0f7,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00906158518046,higher,predicted,0.991863429546 +rcc-vaccine-7c1009a0bb0524b3,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.731387615204,higher,predicted,0.949160933495 +rcc-vaccine-42e869aba90dc4bd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.219947695732,higher,predicted,0.773827433586 +rcc-vaccine-a1a88d1c11b4fcdd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.757146000862,higher,predicted,0.971038460732 +rcc-vaccine-d42b9fb3a44366ac,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.418166160583,higher,predicted,0.998265862465 +rcc-vaccine-ce806d8b1a4132b2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.172158882022,higher,predicted,0.996777057648 +rcc-vaccine-ea51c18cc4e16995,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.731171905994,higher,predicted,0.997755050659 +rcc-vaccine-9bce007ba3a4aff0,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.460265338421,higher,predicted,0.826100230217 +rcc-vaccine-aa3019acbdddb7dc,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.657150745392,higher,predicted,0.952590823174 +rcc-vaccine-1bcdaed331509a08,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.172158882022,higher,predicted,0.996777057648 +rcc-vaccine-4a1853322eea7a14,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.461393415928,higher,predicted,0.986970543861 +rcc-vaccine-cde49959d75e6ec2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.75685107708,higher,predicted,0.979090809822 +rcc-vaccine-7f8e1f8e02332663,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0762795731425,higher,predicted,0.997413814068 +rcc-vaccine-b8aa4d97d214f765,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.990990817547,higher,predicted,0.999659538269 +rcc-vaccine-5993797f7a6c7057,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.442390143871,higher,predicted,0.993007063866 +rcc-vaccine-eacb2f310488ca9d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0443214699626,higher,predicted,0.999183535576 +rcc-vaccine-70f05fcfa038224f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.644423604012,higher,predicted,0.974320590496 +rcc-vaccine-4d15c4e62bb19311,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00129490566906,higher,predicted,0.908160805702 +rcc-vaccine-a6b2180415ff1b56,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.103126898408,higher,predicted,0.870270550251 +rcc-vaccine-e52f3609d694b4ba,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.249618858099,higher,predicted,0.946961700916 +rcc-vaccine-763f29ff7ac4c65d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0525496602058,higher,predicted,0.99926173687 +rcc-vaccine-c331f65899396b9b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.976519405842,higher,predicted,0.984708964825 +rcc-vaccine-09c9b699d977f421,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.816194355488,higher,predicted,0.991399765015 +rcc-vaccine-e944d5a782a7d074,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.533009707928,higher,predicted,0.962785363197 +rcc-vaccine-75f86cf28dca60e2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0963276177645,higher,predicted,0.998524844646 +rcc-vaccine-5b2ef450eaf2248a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.985190093517,higher,predicted,0.999938488007 +rcc-vaccine-c50e7f0e1b3b69a5,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0859711095691,higher,predicted,0.991966068745 +rcc-vaccine-ae70f68909b113e8,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00732829514891,higher,predicted,0.967723369598 +rcc-vaccine-efdf21635c142ab8,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.00268213148229,higher,predicted,0.965277791023 +rcc-vaccine-817b1ac6328997a7,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.920276165009,higher,predicted,0.99510204792 +rcc-vaccine-feb66d3c66c8a4de,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.548630595207,higher,predicted,0.991096198559 +rcc-vaccine-7946e9a10fe114cf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.313266456127,higher,predicted,0.974083065987 +rcc-vaccine-f3e32e857dd09d71,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.02863131091,higher,predicted,0.999120116234 +rcc-vaccine-240bf5b232815c6a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.965816140175,higher,predicted,0.999469399452 +rcc-vaccine-40c01a17a3ed4a2e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.660215973854,higher,predicted,0.978103756905 +rcc-vaccine-7b823cc728ea1450,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.93551158905,higher,predicted,0.962181270123 +rcc-vaccine-0c7eb5e3295f0ad2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0372959189117,higher,predicted,0.999003231525 +rcc-vaccine-88aaf2311ce63196,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.959538161755,higher,predicted,0.999945759773 +rcc-vaccine-bcae618cfad77c1f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.255532085896,higher,predicted,0.933230400085 +rcc-vaccine-11f61d47947266cc,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.994814693928,higher,predicted,0.990790724754 +rcc-vaccine-f4a4db2362e7a54e,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.576282143593,higher,predicted,0.998294651508 +rcc-vaccine-1eceb5b6a969d721,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.721192061901,higher,predicted,0.99988758564 +rcc-vaccine-55106745d24cfbf4,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.676800429821,higher,predicted,0.945376694202 +rcc-vaccine-a808947689fbe42c,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.800811767578,higher,predicted,0.99976503849 +rcc-vaccine-590f5ea08f24e2cf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.528627693653,higher,predicted,0.999486088753 +rcc-vaccine-b984dc3be5c1fa17,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.576282143593,higher,predicted,0.998294651508 +rcc-vaccine-6ff6e758ac279425,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.5315746665,higher,predicted,0.868035018444 +rcc-vaccine-aa0d4e0503dacba4,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.810282886028,higher,predicted,0.994153857231 +rcc-vaccine-4d44829f31cf5558,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.786226093769,higher,predicted,0.988708198071 +rcc-vaccine-4f7d9f9f22d48440,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.938049137592,higher,predicted,0.999858498573 +rcc-vaccine-60d1ba1b8ae350fd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.907981276512,higher,predicted,0.999875664711 +rcc-vaccine-7ec85836cb09ee30,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.567132294178,higher,predicted,0.990272343159 +rcc-vaccine-4b7fdcc54f65b98b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.907981276512,higher,predicted,0.999875664711 +rcc-vaccine-d02de725ecc685a5,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.716995000839,higher,predicted,0.998158454895 +rcc-vaccine-e1d984ebf245728f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.777408897877,higher,predicted,0.999284625053 +rcc-vaccine-a9a3c2bc98a74746,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0750500410795,higher,predicted,0.91932952404 +rcc-vaccine-da610226e338a8de,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.465691566467,higher,predicted,0.999276459217 +rcc-vaccine-ca0a8a8bb4e2d094,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.952795028687,higher,predicted,0.997379779816 +rcc-vaccine-c45585f253341070,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.494304269552,higher,predicted,0.996902823448 +rcc-vaccine-b2761351ccb757b7,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0750500410795,higher,predicted,0.91932952404 +rcc-vaccine-b1ad936ccc2f249a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.494304269552,higher,predicted,0.996902823448 +rcc-vaccine-02261377bb800592,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.129811197519,higher,predicted,0.999617755413 +rcc-vaccine-834aff6d2a7dc363,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.88037699461,higher,predicted,0.955722689629 +rcc-vaccine-0036b8c45a4cd20d,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.0750500410795,higher,predicted,0.91932952404 +rcc-vaccine-08447808dd0300dd,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.494304269552,higher,predicted,0.996902823448 +rcc-vaccine-34c3e7e7a2dcd8b6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.129811197519,higher,predicted,0.999617755413 +rcc-vaccine-1ae968c5ae01f49b,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.88037699461,higher,predicted,0.955722689629 +rcc-vaccine-29f7c92d094c1ced,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.165366202593,higher,predicted,0.998827159405 +rcc-vaccine-0aaef5904890371c,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.88037699461,higher,predicted,0.955722689629 +rcc-vaccine-27b47d43dab2b353,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.952636361122,higher,predicted,0.999851703644 +rcc-vaccine-46b486b9d199e93a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.814331412315,higher,predicted,0.790807783604 +rcc-vaccine-b8a9c44bcf53568f,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.161298811436,higher,predicted,0.998297870159 +rcc-vaccine-7425139a6230adc0,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.889500260353,higher,predicted,0.977608501911 +rcc-vaccine-e39c49c350f42009,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.468123346567,higher,predicted,0.999036669731 +rcc-vaccine-73717d4619b66faf,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.963266253471,higher,predicted,0.998651981354 +rcc-vaccine-f130cbb5ea1ca0fa,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.99343174696,higher,predicted,0.916508376598 +rcc-vaccine-3f16099c911d2791,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.889500260353,higher,predicted,0.977608501911 +rcc-vaccine-07bc8054e1edd65a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.314443677664,higher,predicted,0.999527335167 +rcc-vaccine-99bd603a759d6b22,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.950844585896,higher,predicted,0.970935940742 +rcc-vaccine-ad726d8ef91bd0e6,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.969846844673,higher,predicted,0.992808938026 +rcc-vaccine-972e1f28fe4cb96a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.678239226341,higher,predicted,0.980034708977 +rcc-vaccine-e0d628668c1eccf2,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.76940625906,higher,predicted,0.512591063976 +rcc-vaccine-3d165d772065bb3a,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.982267737389,higher,predicted,0.99388551712 +rcc-vaccine-9ef49682993d2ea1,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.435784041882,higher,predicted,0.955987811089 +rcc-vaccine-6c01a27a6a575007,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.963434100151,higher,predicted,0.847052097321 +rcc-vaccine-12b152d35907e280,DeepHLApan,1.1.1@ac1f4beb,immunogenicity,0.205441281199,higher,predicted,0.999481081963 diff --git a/results/raw_predictions/rcc/deepimmuno-cnn.csv b/results/raw_predictions/rcc/deepimmuno-cnn.csv new file mode 100644 index 0000000..d6e548c --- /dev/null +++ b/results/raw_predictions/rcc/deepimmuno-cnn.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status +rcc-vaccine-8c2e1ec6d8f675cf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.936987876892,higher,predicted +rcc-vaccine-6a6ffb8b81ec9208,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-4adeedbe0e35cf09,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-e06799bdf8de8af5,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.936987876892,higher,predicted +rcc-vaccine-969998ab9d8b4dc1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-a5952e9e71bd27bd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.783586323261,higher,predicted +rcc-vaccine-6fa3e9ad450e030e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.936987876892,higher,predicted +rcc-vaccine-8a9041cc55b08e38,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.697320222855,higher,predicted +rcc-vaccine-e11c81feedc5751e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-fefffdcccca8649c,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.697254955769,higher,predicted +rcc-vaccine-940b62df9c3078c4,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-43742d14e082c2e6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-769c6b305827c570,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-e74f183589095f61,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-15ab70cb934aad8f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-c8a8ce5ab15f9d4b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.688341677189,higher,predicted +rcc-vaccine-f05602883865f4bf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-b92105a12fb6f028,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-a004075aed453bcf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-70eb710f90e2973b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-c81528933be4a147,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.374690830708,higher,predicted +rcc-vaccine-2a2f0b8f959aa3f6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-74e8a99b7ef86c61,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-88bea2527dd72a1d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.985417187214,higher,predicted +rcc-vaccine-ff6bbb8d69468a47,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-8ea6c4f21d2c81ee,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-b2f6110d808ac0da,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.838476061821,higher,predicted +rcc-vaccine-f43e93564087f160,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.8036480546,higher,predicted +rcc-vaccine-fc1276f53c3f45e1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.904640376568,higher,predicted +rcc-vaccine-3e6b1c1d8de0ce71,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.985883593559,higher,predicted +rcc-vaccine-4255bdded9e984bd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-31c3a5a1c962db43,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.761615693569,higher,predicted +rcc-vaccine-d3d4046d155047b6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-5e4096b57e93c92e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.408737450838,higher,predicted +rcc-vaccine-e8d6aeb51705642a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.988444507122,higher,predicted +rcc-vaccine-26dc24e186221fe2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.945877850056,higher,predicted +rcc-vaccine-364822c022710958,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-9c477d13a3359f5e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-487a8e43d12c17ac,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-539b50b223ea0ee1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-618d6aa7d8232657,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-71f5d2ab19a389e8,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.309658080339,higher,predicted +rcc-vaccine-409a0661a319bbb1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-9f431d040990f0f7,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.302717536688,higher,predicted +rcc-vaccine-7c1009a0bb0524b3,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-42e869aba90dc4bd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-a1a88d1c11b4fcdd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-d42b9fb3a44366ac,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-ce806d8b1a4132b2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.422743946314,higher,predicted +rcc-vaccine-ea51c18cc4e16995,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-9bce007ba3a4aff0,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-aa3019acbdddb7dc,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-1bcdaed331509a08,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.422743946314,higher,predicted +rcc-vaccine-4a1853322eea7a14,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-cde49959d75e6ec2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-7f8e1f8e02332663,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-b8aa4d97d214f765,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-5993797f7a6c7057,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-eacb2f310488ca9d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-70f05fcfa038224f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-4d15c4e62bb19311,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-a6b2180415ff1b56,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-e52f3609d694b4ba,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-763f29ff7ac4c65d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-c331f65899396b9b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-09c9b699d977f421,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-e944d5a782a7d074,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-75f86cf28dca60e2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-5b2ef450eaf2248a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-c50e7f0e1b3b69a5,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.973050177097,higher,predicted +rcc-vaccine-ae70f68909b113e8,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.597477078438,higher,predicted +rcc-vaccine-efdf21635c142ab8,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-817b1ac6328997a7,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-feb66d3c66c8a4de,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-7946e9a10fe114cf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-f3e32e857dd09d71,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-240bf5b232815c6a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-40c01a17a3ed4a2e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-7b823cc728ea1450,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-0c7eb5e3295f0ad2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-88aaf2311ce63196,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-bcae618cfad77c1f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.365942507982,higher,predicted +rcc-vaccine-11f61d47947266cc,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-f4a4db2362e7a54e,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.577033996582,higher,predicted +rcc-vaccine-1eceb5b6a969d721,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-55106745d24cfbf4,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.356880515814,higher,predicted +rcc-vaccine-a808947689fbe42c,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.917404770851,higher,predicted +rcc-vaccine-590f5ea08f24e2cf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.893288612366,higher,predicted +rcc-vaccine-b984dc3be5c1fa17,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.577033996582,higher,predicted +rcc-vaccine-6ff6e758ac279425,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.85473293066,higher,predicted +rcc-vaccine-aa0d4e0503dacba4,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.729778409004,higher,predicted +rcc-vaccine-4d44829f31cf5558,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.862372934818,higher,predicted +rcc-vaccine-4f7d9f9f22d48440,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-60d1ba1b8ae350fd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.91576641798,higher,predicted +rcc-vaccine-7ec85836cb09ee30,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-4b7fdcc54f65b98b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.91576641798,higher,predicted +rcc-vaccine-d02de725ecc685a5,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.730106890202,higher,predicted +rcc-vaccine-e1d984ebf245728f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.892051696777,higher,predicted +rcc-vaccine-a9a3c2bc98a74746,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.473661601543,higher,predicted +rcc-vaccine-da610226e338a8de,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.803110301495,higher,predicted +rcc-vaccine-ca0a8a8bb4e2d094,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-c45585f253341070,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.964553833008,higher,predicted +rcc-vaccine-b2761351ccb757b7,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.473661601543,higher,predicted +rcc-vaccine-b1ad936ccc2f249a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.964553833008,higher,predicted +rcc-vaccine-02261377bb800592,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-834aff6d2a7dc363,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-0036b8c45a4cd20d,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.473661601543,higher,predicted +rcc-vaccine-08447808dd0300dd,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.964553833008,higher,predicted +rcc-vaccine-34c3e7e7a2dcd8b6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-1ae968c5ae01f49b,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-29f7c92d094c1ced,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-0aaef5904890371c,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-27b47d43dab2b353,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-46b486b9d199e93a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-b8a9c44bcf53568f,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.853769600391,higher,predicted +rcc-vaccine-7425139a6230adc0,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.795969843864,higher,predicted +rcc-vaccine-e39c49c350f42009,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.694881081581,higher,predicted +rcc-vaccine-73717d4619b66faf,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-f130cbb5ea1ca0fa,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-3f16099c911d2791,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.795969843864,higher,predicted +rcc-vaccine-07bc8054e1edd65a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.558099448681,higher,predicted +rcc-vaccine-99bd603a759d6b22,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-ad726d8ef91bd0e6,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_length +rcc-vaccine-972e1f28fe4cb96a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.941862821579,higher,predicted +rcc-vaccine-e0d628668c1eccf2,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-3d165d772065bb3a,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-9ef49682993d2ea1,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.355489432812,higher,predicted +rcc-vaccine-6c01a27a6a575007,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,,higher,unsupported_hla +rcc-vaccine-12b152d35907e280,DeepImmuno-CNN,1.0@df42ac5b,immunogenicity,0.2018879354,higher,predicted diff --git a/results/raw_predictions/rcc/prime-2.0.csv b/results/raw_predictions/rcc/prime-2.0.csv new file mode 100644 index 0000000..e3ea1fc --- /dev/null +++ b/results/raw_predictions/rcc/prime-2.0.csv @@ -0,0 +1,130 @@ +record_id,predictor,predictor_version,task,score,score_direction,status,prime_percent_rank,mixmhcpred_percent_rank,allele +rcc-vaccine-8c2e1ec6d8f675cf,PRIME,2.0,immunogenicity,0.231349,higher,predicted,0.013,0.002,B5701 +rcc-vaccine-6a6ffb8b81ec9208,PRIME,2.0,immunogenicity,0.031043,higher,predicted,0.873,0.237,B4001 +rcc-vaccine-4adeedbe0e35cf09,PRIME,2.0,immunogenicity,,higher,unsupported_hla,,,C0602HLAA6601 +rcc-vaccine-e06799bdf8de8af5,PRIME,2.0,immunogenicity,0.231349,higher,predicted,0.013,0.002,B5701 +rcc-vaccine-969998ab9d8b4dc1,PRIME,2.0,immunogenicity,0.003103,higher,predicted,13.755,7.408,A6601 +rcc-vaccine-a5952e9e71bd27bd,PRIME,2.0,immunogenicity,0.021044,higher,predicted,1.546,1.758,B4001 +rcc-vaccine-6fa3e9ad450e030e,PRIME,2.0,immunogenicity,0.231349,higher,predicted,0.013,0.002,B5701 +rcc-vaccine-8a9041cc55b08e38,PRIME,2.0,immunogenicity,0.073447,higher,predicted,0.21,0.456,B4001 +rcc-vaccine-e11c81feedc5751e,PRIME,2.0,immunogenicity,0.000241,higher,predicted,68.331,20.277,A6601 +rcc-vaccine-fefffdcccca8649c,PRIME,2.0,immunogenicity,0.142751,higher,predicted,0.057,0.109,B4001 +rcc-vaccine-940b62df9c3078c4,PRIME,2.0,immunogenicity,0.173992,higher,predicted,0.041,0.075,A6601 +rcc-vaccine-43742d14e082c2e6,PRIME,2.0,immunogenicity,0.000881,higher,predicted,33.872,21.201,A6601 +rcc-vaccine-769c6b305827c570,PRIME,2.0,immunogenicity,0.01609,higher,predicted,2.513,0.657,B3503 +rcc-vaccine-e74f183589095f61,PRIME,2.0,immunogenicity,0.000172,higher,predicted,79.645,68.608,A0201 +rcc-vaccine-15ab70cb934aad8f,PRIME,2.0,immunogenicity,0.200487,higher,predicted,0.036,0.195,B3801 +rcc-vaccine-c8a8ce5ab15f9d4b,PRIME,2.0,immunogenicity,0.002497,higher,predicted,17.027,7.799,A0201 +rcc-vaccine-f05602883865f4bf,PRIME,2.0,immunogenicity,0.05337,higher,predicted,0.552,0.384,B3503 +rcc-vaccine-b92105a12fb6f028,PRIME,2.0,immunogenicity,0.211211,higher,predicted,0.03,0.025,B3801 +rcc-vaccine-a004075aed453bcf,PRIME,2.0,immunogenicity,0.002451,higher,predicted,16.927,2.108,B3501 +rcc-vaccine-70eb710f90e2973b,PRIME,2.0,immunogenicity,0.009664,higher,predicted,4.304,1.428,B3503 +rcc-vaccine-c81528933be4a147,PRIME,2.0,immunogenicity,0.231073,higher,predicted,0.013,0.04,A0101 +rcc-vaccine-2a2f0b8f959aa3f6,PRIME,2.0,immunogenicity,0.015484,higher,predicted,3.294,0.526,C0701 +rcc-vaccine-74e8a99b7ef86c61,PRIME,2.0,immunogenicity,0.099002,higher,predicted,0.199,0.162,B0801 +rcc-vaccine-88bea2527dd72a1d,PRIME,2.0,immunogenicity,0.0243,higher,predicted,2.529,5.059,C0602 +rcc-vaccine-ff6bbb8d69468a47,PRIME,2.0,immunogenicity,0.00482,higher,predicted,10.393,1.108,C0602 +rcc-vaccine-8ea6c4f21d2c81ee,PRIME,2.0,immunogenicity,0.01207,higher,predicted,3.638,0.788,B1302 +rcc-vaccine-b2f6110d808ac0da,PRIME,2.0,immunogenicity,0.108979,higher,predicted,0.141,0.117,A3001 +rcc-vaccine-f43e93564087f160,PRIME,2.0,immunogenicity,0.059356,higher,predicted,0.927,0.03,C0602 +rcc-vaccine-fc1276f53c3f45e1,PRIME,2.0,immunogenicity,0.00163,higher,predicted,21.154,11.829,A3001 +rcc-vaccine-3e6b1c1d8de0ce71,PRIME,2.0,immunogenicity,0.135059,higher,predicted,0.247,0.839,C0602 +rcc-vaccine-4255bdded9e984bd,PRIME,2.0,immunogenicity,0.14839,higher,predicted,0.16,0.078,C0701 +rcc-vaccine-31c3a5a1c962db43,PRIME,2.0,immunogenicity,0.105933,higher,predicted,0.174,0.157,B0801 +rcc-vaccine-d3d4046d155047b6,PRIME,2.0,immunogenicity,0.225538,higher,predicted,0.016,0.009,B1302 +rcc-vaccine-5e4096b57e93c92e,PRIME,2.0,immunogenicity,0.137711,higher,predicted,0.085,0.204,A0101 +rcc-vaccine-e8d6aeb51705642a,PRIME,2.0,immunogenicity,0.123093,higher,predicted,0.299,0.205,C0602 +rcc-vaccine-26dc24e186221fe2,PRIME,2.0,immunogenicity,0.276251,higher,predicted,0.003,0.026,A3001 +rcc-vaccine-364822c022710958,PRIME,2.0,immunogenicity,0.024145,higher,predicted,1.627,0.172,B0801 +rcc-vaccine-9c477d13a3359f5e,PRIME,2.0,immunogenicity,0.040185,higher,predicted,1.289,0.407,C0701 +rcc-vaccine-487a8e43d12c17ac,PRIME,2.0,immunogenicity,0.02398,higher,predicted,1.639,0.347,B0801 +rcc-vaccine-539b50b223ea0ee1,PRIME,2.0,immunogenicity,0.08652,higher,predicted,0.222,0.032,A0101 +rcc-vaccine-618d6aa7d8232657,PRIME,2.0,immunogenicity,0.012006,higher,predicted,4.131,2.584,C0701 +rcc-vaccine-71f5d2ab19a389e8,PRIME,2.0,immunogenicity,0.072737,higher,predicted,0.301,1.475,A0101 +rcc-vaccine-409a0661a319bbb1,PRIME,2.0,immunogenicity,0.040185,higher,predicted,1.289,0.407,C0701 +rcc-vaccine-9f431d040990f0f7,PRIME,2.0,immunogenicity,0.08679,higher,predicted,0.221,0.36,A0101 +rcc-vaccine-7c1009a0bb0524b3,PRIME,2.0,immunogenicity,0.00103,higher,predicted,29.367,4.658,C0701 +rcc-vaccine-42e869aba90dc4bd,PRIME,2.0,immunogenicity,0.002872,higher,predicted,13.901,11.529,A0101 +rcc-vaccine-a1a88d1c11b4fcdd,PRIME,2.0,immunogenicity,0.000868,higher,predicted,33.067,4.751,C0701 +rcc-vaccine-d42b9fb3a44366ac,PRIME,2.0,immunogenicity,0.08652,higher,predicted,0.222,0.032,A0101 +rcc-vaccine-ce806d8b1a4132b2,PRIME,2.0,immunogenicity,0.16414,higher,predicted,0.054,0.099,A0101 +rcc-vaccine-ea51c18cc4e16995,PRIME,2.0,immunogenicity,0.012006,higher,predicted,4.131,2.584,C0701 +rcc-vaccine-9bce007ba3a4aff0,PRIME,2.0,immunogenicity,0.002137,higher,predicted,18.037,8.154,C0701 +rcc-vaccine-aa3019acbdddb7dc,PRIME,2.0,immunogenicity,0.002911,higher,predicted,14.59,2.151,C0701 +rcc-vaccine-1bcdaed331509a08,PRIME,2.0,immunogenicity,0.16414,higher,predicted,0.054,0.099,A0101 +rcc-vaccine-4a1853322eea7a14,PRIME,2.0,immunogenicity,0.004955,higher,predicted,8.866,6.912,C0701 +rcc-vaccine-cde49959d75e6ec2,PRIME,2.0,immunogenicity,0.010927,higher,predicted,3.78,0.565,B0801 +rcc-vaccine-7f8e1f8e02332663,PRIME,2.0,immunogenicity,0.054186,higher,predicted,0.473,0.167,A0101 +rcc-vaccine-b8aa4d97d214f765,PRIME,2.0,immunogenicity,0.020849,higher,predicted,2.173,0.627,C0303 +rcc-vaccine-5993797f7a6c7057,PRIME,2.0,immunogenicity,0.024739,higher,predicted,1.31,0.764,A0301 +rcc-vaccine-eacb2f310488ca9d,PRIME,2.0,immunogenicity,0.045668,higher,predicted,0.474,0.461,B4001 +rcc-vaccine-70f05fcfa038224f,PRIME,2.0,immunogenicity,0.029104,higher,predicted,1.244,1.311,A3101 +rcc-vaccine-4d15c4e62bb19311,PRIME,2.0,immunogenicity,0.001505,higher,predicted,24.31,7.077,B5501 +rcc-vaccine-a6b2180415ff1b56,PRIME,2.0,immunogenicity,0.104449,higher,predicted,0.137,0.165,B5501 +rcc-vaccine-e52f3609d694b4ba,PRIME,2.0,immunogenicity,0.01842,higher,predicted,1.84,2.946,A0301 +rcc-vaccine-763f29ff7ac4c65d,PRIME,2.0,immunogenicity,0.016938,higher,predicted,1.982,2.051,B4001 +rcc-vaccine-c331f65899396b9b,PRIME,2.0,immunogenicity,0.059575,higher,predicted,0.579,0.238,C0303 +rcc-vaccine-09c9b699d977f421,PRIME,2.0,immunogenicity,0.078423,higher,predicted,0.278,0.423,A3101 +rcc-vaccine-e944d5a782a7d074,PRIME,2.0,immunogenicity,0.00665,higher,predicted,5.507,1.261,B4001 +rcc-vaccine-75f86cf28dca60e2,PRIME,2.0,immunogenicity,0.015183,higher,predicted,2.286,0.295,A0301 +rcc-vaccine-5b2ef450eaf2248a,PRIME,2.0,immunogenicity,0.023547,higher,predicted,1.908,0.479,C0303 +rcc-vaccine-c50e7f0e1b3b69a5,PRIME,2.0,immunogenicity,0.161118,higher,predicted,0.05,0.099,A0301 +rcc-vaccine-ae70f68909b113e8,PRIME,2.0,immunogenicity,0.122524,higher,predicted,0.079,0.728,B4001 +rcc-vaccine-efdf21635c142ab8,PRIME,2.0,immunogenicity,0.006965,higher,predicted,5.891,1,B5501 +rcc-vaccine-817b1ac6328997a7,PRIME,2.0,immunogenicity,0.016054,higher,predicted,2.784,0.379,A3201 +rcc-vaccine-feb66d3c66c8a4de,PRIME,2.0,immunogenicity,0.017987,higher,predicted,2.542,0.656,A0201 +rcc-vaccine-7946e9a10fe114cf,PRIME,2.0,immunogenicity,0.005165,higher,predicted,8.549,3.652,B4402 +rcc-vaccine-f3e32e857dd09d71,PRIME,2.0,immunogenicity,0.065268,higher,predicted,0.351,0.291,B4402 +rcc-vaccine-240bf5b232815c6a,PRIME,2.0,immunogenicity,0.189752,higher,predicted,0.039,0.109,C0501 +rcc-vaccine-40c01a17a3ed4a2e,PRIME,2.0,immunogenicity,0.022939,higher,predicted,1.927,0.689,A0201 +rcc-vaccine-7b823cc728ea1450,PRIME,2.0,immunogenicity,0.104224,higher,predicted,0.224,0.03,A3201 +rcc-vaccine-0c7eb5e3295f0ad2,PRIME,2.0,immunogenicity,0.027787,higher,predicted,1.291,0.125,B4402 +rcc-vaccine-88aaf2311ce63196,PRIME,2.0,immunogenicity,0.008837,higher,predicted,4.523,2.281,C0501 +rcc-vaccine-bcae618cfad77c1f,PRIME,2.0,immunogenicity,0.022536,higher,predicted,1.692,7.234,B4402 +rcc-vaccine-11f61d47947266cc,PRIME,2.0,immunogenicity,0.18811,higher,predicted,0.047,0.002,A3201 +rcc-vaccine-f4a4db2362e7a54e,PRIME,2.0,immunogenicity,0.201591,higher,predicted,0.04,0.049,A0201 +rcc-vaccine-1eceb5b6a969d721,PRIME,2.0,immunogenicity,0.038765,higher,predicted,0.852,0.241,B2705 +rcc-vaccine-55106745d24cfbf4,PRIME,2.0,immunogenicity,0.034507,higher,predicted,1.225,0.327,A0201 +rcc-vaccine-a808947689fbe42c,PRIME,2.0,immunogenicity,0.021044,higher,predicted,1.919,0.811,B2705 +rcc-vaccine-590f5ea08f24e2cf,PRIME,2.0,immunogenicity,0.029875,higher,predicted,1.254,1.004,B2705 +rcc-vaccine-b984dc3be5c1fa17,PRIME,2.0,immunogenicity,0.201591,higher,predicted,0.04,0.049,A0201 +rcc-vaccine-6ff6e758ac279425,PRIME,2.0,immunogenicity,0.060779,higher,predicted,0.559,1.106,A0201 +rcc-vaccine-aa0d4e0503dacba4,PRIME,2.0,immunogenicity,0.010168,higher,predicted,4.331,4.754,B2705 +rcc-vaccine-4d44829f31cf5558,PRIME,2.0,immunogenicity,0.070594,higher,predicted,0.343,0.943,B2705 +rcc-vaccine-4f7d9f9f22d48440,PRIME,2.0,immunogenicity,0.056781,higher,predicted,0.482,0.285,B2705 +rcc-vaccine-60d1ba1b8ae350fd,PRIME,2.0,immunogenicity,0.148328,higher,predicted,0.077,0.017,B2705 +rcc-vaccine-7ec85836cb09ee30,PRIME,2.0,immunogenicity,0.006543,higher,predicted,6.468,1.956,B2706 +rcc-vaccine-4b7fdcc54f65b98b,PRIME,2.0,immunogenicity,0.148328,higher,predicted,0.077,0.017,B2705 +rcc-vaccine-d02de725ecc685a5,PRIME,2.0,immunogenicity,0.116072,higher,predicted,0.136,0.078,B2705 +rcc-vaccine-e1d984ebf245728f,PRIME,2.0,immunogenicity,0.024212,higher,predicted,1.647,2.152,B2705 +rcc-vaccine-a9a3c2bc98a74746,PRIME,2.0,immunogenicity,0.134396,higher,predicted,0.079,0.206,B4402 +rcc-vaccine-da610226e338a8de,PRIME,2.0,immunogenicity,0.084327,higher,predicted,0.336,0.212,A0201 +rcc-vaccine-ca0a8a8bb4e2d094,PRIME,2.0,immunogenicity,0.004759,higher,predicted,8.165,3.166,C0501 +rcc-vaccine-c45585f253341070,PRIME,2.0,immunogenicity,0.271417,higher,predicted,0.005,0.008,B0801 +rcc-vaccine-b2761351ccb757b7,PRIME,2.0,immunogenicity,0.134396,higher,predicted,0.079,0.206,B4402 +rcc-vaccine-b1ad936ccc2f249a,PRIME,2.0,immunogenicity,0.271417,higher,predicted,0.005,0.008,B0801 +rcc-vaccine-02261377bb800592,PRIME,2.0,immunogenicity,0.010341,higher,predicted,4.173,0.359,B4402 +rcc-vaccine-834aff6d2a7dc363,PRIME,2.0,immunogenicity,0.028399,higher,predicted,1.251,0.374,B4402 +rcc-vaccine-0036b8c45a4cd20d,PRIME,2.0,immunogenicity,0.134396,higher,predicted,0.079,0.206,B4402 +rcc-vaccine-08447808dd0300dd,PRIME,2.0,immunogenicity,0.271417,higher,predicted,0.005,0.008,B0801 +rcc-vaccine-34c3e7e7a2dcd8b6,PRIME,2.0,immunogenicity,0.010341,higher,predicted,4.173,0.359,B4402 +rcc-vaccine-1ae968c5ae01f49b,PRIME,2.0,immunogenicity,0.028399,higher,predicted,1.251,0.374,B4402 +rcc-vaccine-29f7c92d094c1ced,PRIME,2.0,immunogenicity,0.054551,higher,predicted,0.469,0.085,B4402 +rcc-vaccine-0aaef5904890371c,PRIME,2.0,immunogenicity,0.028399,higher,predicted,1.251,0.374,B4402 +rcc-vaccine-27b47d43dab2b353,PRIME,2.0,immunogenicity,0.072271,higher,predicted,0.374,0.168,C0501 +rcc-vaccine-46b486b9d199e93a,PRIME,2.0,immunogenicity,0.078618,higher,predicted,0.525,0.319,C0701 +rcc-vaccine-b8a9c44bcf53568f,PRIME,2.0,immunogenicity,0.18383,higher,predicted,0.032,0.042,A0301 +rcc-vaccine-7425139a6230adc0,PRIME,2.0,immunogenicity,0.154046,higher,predicted,0.065,0.062,B1801 +rcc-vaccine-e39c49c350f42009,PRIME,2.0,immunogenicity,0.046887,higher,predicted,0.435,0.03,B0702 +rcc-vaccine-73717d4619b66faf,PRIME,2.0,immunogenicity,0.00442,higher,predicted,9.373,2.108,C0702 +rcc-vaccine-f130cbb5ea1ca0fa,PRIME,2.0,immunogenicity,0.034846,higher,predicted,1.486,1.968,C1203 +rcc-vaccine-3f16099c911d2791,PRIME,2.0,immunogenicity,0.154046,higher,predicted,0.065,0.062,B1801 +rcc-vaccine-07bc8054e1edd65a,PRIME,2.0,immunogenicity,0.123231,higher,predicted,0.071,0.161,B0702 +rcc-vaccine-99bd603a759d6b22,PRIME,2.0,immunogenicity,0.026807,higher,predicted,1.73,0.405,C0702 +rcc-vaccine-ad726d8ef91bd0e6,PRIME,2.0,immunogenicity,0.008608,higher,predicted,5.148,0.839,C0702 +rcc-vaccine-972e1f28fe4cb96a,PRIME,2.0,immunogenicity,0.053712,higher,predicted,0.499,0.324,B1801 +rcc-vaccine-e0d628668c1eccf2,PRIME,2.0,immunogenicity,0.015526,higher,predicted,4.37,1.203,A2501 +rcc-vaccine-3d165d772065bb3a,PRIME,2.0,immunogenicity,0.118113,higher,predicted,0.253,0.026,C1203 +rcc-vaccine-9ef49682993d2ea1,PRIME,2.0,immunogenicity,0.051028,higher,predicted,0.381,0.074,B0702 +rcc-vaccine-6c01a27a6a575007,PRIME,2.0,immunogenicity,0.049132,higher,predicted,0.842,0.272,C0702 +rcc-vaccine-12b152d35907e280,PRIME,2.0,immunogenicity,0.066229,higher,predicted,0.249,0.553,B0702 diff --git a/scripts/analyze_stability.py b/scripts/analyze_stability.py new file mode 100755 index 0000000..a8aea1c --- /dev/null +++ b/scripts/analyze_stability.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 +"""Descriptive cross-dataset stability and model-selection-risk analysis. + +This intentionally treats fixed predictors as descriptive/exploratory. It does +not fit models, test causal hypotheses, or relabel unsupported predictions. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import math +import random +from collections import defaultdict +from itertools import combinations +from pathlib import Path + +KS = (1, 5, 10, 20) + + +def read_csv(p): + with Path(p).open(newline="", encoding="utf-8-sig") as f: + return list(csv.DictReader(f)) + + +def auc(rows): + x = sorted((float(r["score"]), int(r["label"])) for r in rows) + pos = sum(y for _, y in x) + neg = len(x) - pos + if not pos or not neg: + return None + rank = 0 + s = 0 + i = 0 + while i < len(x): + j = i + 1 + while j < len(x) and x[j][0] == x[i][0]: + j += 1 + rank = (i + j + 1) / 2 + s += rank * sum(y for _, y in x[i:j]) + i = j + return (s - pos * (pos + 1) / 2) / (pos * neg) + + +def patient_metric(rows, k): + groups = defaultdict(list) + for r in rows: + groups[r["patient_id"]].append(r) + vals = [] + for rs in groups.values(): + pos = sum(int(r["label"]) for r in rs) + if not pos: + continue + rs = sorted(rs, key=lambda r: -float(r["score"])) + top = rs[: min(k, len(rs))] + hits = sum(int(r["label"]) for r in top) + vals.append(hits / pos) + return sum(vals) / len(vals) if vals else None + + +def spearman(a, b): + def rank(v): + order = sorted(range(len(v)), key=v.__getitem__) + out = [0.0] * len(v) + i = 0 + while i < len(v): + j = i + 1 + while j < len(v) and v[order[j]] == v[order[i]]: + j += 1 + z = (i + j + 1) / 2 + for q in order[i:j]: + out[q] = z + i = j + return out + + if len(a) < 2: + return None + ra, rb = rank(a), rank(b) + ma = sum(ra) / len(ra) + mb = sum(rb) / len(rb) + num = sum((x - ma) * (y - mb) for x, y in zip(ra, rb)) + den = math.sqrt(sum((x - ma) ** 2 for x in ra) * sum((y - mb) ** 2 for y in rb)) + return num / den if den else None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--output-dir", type=Path, required=True) + ap.add_argument("--bootstrap", type=int, default=2000) + ap.add_argument("--seed", type=int, default=20260820) + ap.add_argument("--coverage", nargs="+", type=float, default=[0.5, 0.8, 0.95]) + ap.add_argument("--benchmark", nargs="+", required=True) + ap.add_argument("--predictions", nargs="+") + ap.add_argument("--prediction-dir", nargs="+") + args = ap.parse_args() + if args.prediction_dir and len(args.prediction_dir) != len(args.benchmark): + ap.error("prediction-dir count must match benchmark count") + if not args.prediction_dir and ( + not args.predictions or len(args.benchmark) != len(args.predictions) + ): + ap.error("provide aligned predictions or prediction-dir per benchmark") + datasets = {} + meta = {} + for idx, bp in enumerate(args.benchmark): + prediction_paths = ( + sorted(Path(args.prediction_dir[idx]).glob("*.csv")) + if args.prediction_dir + else [Path(args.predictions[idx])] + ) + for pp in prediction_paths: + b = {r["record_id"]: r for r in read_csv(bp)} + prs = read_csv(pp) + pred = prs[0]["predictor"] + rows = [] + status = defaultdict(int) + for p in prs: + status[p["status"]] += 1 + if p["status"] == "predicted": + r = b[p["record_id"]] + score = float(p["score"]) + score = score if p["score_direction"] == "higher" else -score + rows.append( + { + "record_id": p["record_id"], + "patient_id": r["patient_id"], + "study_id": r["study_id"], + "hla": r["hla"], + "label": int(r["immunogenicity"]), + "score": score, + } + ) + datasets.setdefault(Path(bp).stem, {})[pred] = rows + meta[pred] = { + "task": prs[0]["task"], + "version": prs[0]["predictor_version"], + "status": dict(status), + } + out = args.output_dir + out.mkdir(parents=True, exist_ok=True) + matrix = [] + rank_rows = [] + first_rows = [] + sens = [] + lodo = [] + domain_meta = [] + for ds, preds in datasets.items(): + supports = [{r["record_id"] for r in rs} for rs in preds.values()] + common = set.intersection(*supports) + benchmark_rows = read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds)) + domain_meta.append( + { + "dataset": ds, + "endpoint": benchmark_rows[0].get("assay_type", "unknown"), + "clinical_context": benchmark_rows[0].get("clinical_context", "unknown"), + "n_records": len(benchmark_rows), + "n_patients": len({r["patient_id"] for r in benchmark_rows}), + "n_studies": len({r["study_id"] for r in benchmark_rows}), + } + ) + for pred, rs in preds.items(): + rr = [r for r in rs if r["record_id"] in common] + matrix += [ + { + "dataset": ds, + "predictor": pred, + "metric": "AUROC", + "value": auc(rr), + "n": len(rr), + "coverage": len(rr) + / len(read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds))), + } + ] + for k in KS: + matrix.append( + { + "dataset": ds, + "predictor": pred, + "metric": f"Recall@{k}", + "value": patient_metric(rr, k), + "n": len(rr), + "coverage": len(rr) + / len(read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds))), + } + ) + study_ids = sorted({r["study_id"] for r in benchmark_rows}) + for held_out in study_ids: + for pred, rs in preds.items(): + rr = [r for r in rs if r["record_id"] in common and r["study_id"] != held_out] + lodo.append( + { + "dataset": ds, + "held_out_domain": held_out, + "predictor": pred, + "metric": "AUROC", + "value": auc(rr), + "n": len(rr), + "patients": len({r["patient_id"] for r in rr}), + "analysis_type": "descriptive_leave_one_domain_out", + } + ) + pairs = list(combinations(sorted(preds), 2)) + byid = {p: {r["record_id"]: r for r in preds[p]} for p in preds} + for a, bp in pairs: + ids = sorted(set(byid[a]) & set(byid[bp])) + ra = [byid[a][i] for i in ids] + rb = [byid[bp][i] for i in ids] + rank_rows.append( + { + "dataset": ds, + "left": a, + "right": bp, + "metric": "record_score_spearman", + "value": spearman([r["score"] for r in ra], [r["score"] for r in rb]), + "n": len(ids), + } + ) + for k in KS: + av = patient_metric(ra, k) + bv = patient_metric(rb, k) + rank_rows.append( + { + "dataset": ds, + "left": a, + "right": bp, + "metric": f"patient_Recall@{k}_difference", + "value": av - bv if av is not None and bv is not None else None, + "n": len(ids), + } + ) + patients = sorted({r["patient_id"] for r in next(iter(preds.values()))}) + rng = random.Random(args.seed) + wins = defaultdict(int) + reversals = defaultdict(int) + task_groups = defaultdict(list) + for p in preds: + task_groups[meta[p]["task"]].append(p) + for _ in range(args.bootstrap): + draw = rng.choices(patients, k=len(patients)) + vals = {} + for p, rs in preds.items(): + by = defaultdict(list) + for r in rs: + by[r["patient_id"]].append(r) + boot = [] + for i, pt in enumerate(draw): + boot += [{**r, "patient_id": f"{i}:{pt}"} for r in by[pt]] + vals[p] = patient_metric(boot, 5) or float("nan") + for task, group in task_groups.items(): + finite = {p: vals[p] for p in group if math.isfinite(vals[p])} + if finite: + top = max(finite.values()) + [ + wins.__setitem__((task, p), wins[(task, p)] + 1) + for p, v in finite.items() + if v == top + ] + if ( + "BigMHC" in vals + and "PRIME" in vals + and math.isfinite(vals["BigMHC"]) + and math.isfinite(vals["PRIME"]) + ): + reversals["BigMHC_vs_PRIME"] += int((vals["BigMHC"] - vals["PRIME"]) < 0) + for task, group in task_groups.items(): + for p in group: + first_rows.append( + { + "dataset": ds, + "task": task, + "predictor": p, + "metric": "Recall@5", + "probability_first": wins[(task, p)] / args.bootstrap, + } + ) + for key, v in reversals.items(): + sens.append( + { + "dataset": ds, + "comparison": key, + "metric": "probability_BigMHC_below_PRIME", + "value": v / args.bootstrap, + "strategy": "common_support", + "k": 5, + } + ) + for c in args.coverage: + for ds, preds in datasets.items(): + total = len(read_csv(next(bp for bp in args.benchmark if Path(bp).stem == ds))) + eligible = {p: rs for p, rs in preds.items() if len(rs) / total >= c} + eligible_supports = [{r["record_id"] for r in rs} for rs in eligible.values()] + union = set.intersection(*eligible_supports) if eligible_supports else set() + sens.append( + { + "dataset": ds, + "comparison": "coverage_filtered_predictors", + "metric": "common_support_n", + "value": len(union), + "strategy": f"coverage_threshold_{c}", + "predictors_in_support": "|".join(sorted(eligible)), + "k": 5, + } + ) + + def write(name, rows): + if not rows: + return + fields = sorted({k for r in rows for k in r}) + with (out / name).open("w", newline="") as f: + w = csv.DictWriter(f, fieldnames=fields, lineterminator="\n") + w.writeheader() + w.writerows(rows) + + write("dataset_predictor_metric_matrix.csv", matrix) + write("rank_stability.csv", rank_rows) + write("model_selection_first_probability.csv", first_rows) + write("sensitivity_summary.csv", sens) + write("leave_one_domain_out.csv", lodo) + write("endpoint_domain_metadata.csv", domain_meta) + try: + import matplotlib.pyplot as plt + + labels = [f"{r['dataset']}\n{r['endpoint']}" for r in domain_meta] + names = sorted({r["predictor"] for r in matrix}) + vals = [] + for name in names: + vals.append( + [ + next( + ( + float(r["value"]) + for r in matrix + if r["dataset"] == d["dataset"] + and r["predictor"] == name + and r["metric"] == "AUROC" + ), + float("nan"), + ) + for d in domain_meta + ] + ) + fig, ax = plt.subplots(figsize=(8, 3.8)) + im = ax.imshow(vals, aspect="auto", vmin=0, vmax=1, cmap="viridis") + ax.set_yticks(range(len(names)), names) + ax.set_xticks(range(len(labels)), labels, rotation=20, ha="right") + ax.set_title("Exploratory AUROC by endpoint/domain and predictor") + fig.colorbar(im, ax=ax, label="AUROC") + fig.tight_layout() + fig.savefig(out / "endpoint_domain_auroc.png", dpi=180) + fig.savefig(out / "endpoint_domain_auroc.svg") + plt.close(fig) + except (ImportError, ModuleNotFoundError) as exc: + (out / "visualization_error.txt").write_text(f"matplotlib unavailable: {exc}\n") + (out / "analysis_metadata.json").write_text( + json.dumps( + { + "analysis_type": "exploratory_descriptive_heterogeneity", + "bootstrap": args.bootstrap, + "seed": args.seed, + "ks": KS, + "coverage_thresholds": args.coverage, + "datasets": list(datasets), + "predictors": meta, + "limitations": [ + "fixed pretrained scores; no causal inference", + "model-selection probabilities are conditional on observed patient samples", + "coverage strategies are descriptive and not multiplicity-adjusted", + ], + }, + indent=2, + ) + + "\n" + ) + print( + json.dumps( + { + "output_dir": str(out), + "datasets": list(datasets), + "matrix_rows": len(matrix), + "bootstrap": args.bootstrap, + } + ) + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/audit_release_boundary.py b/scripts/audit_release_boundary.py new file mode 100755 index 0000000..e4bbd51 --- /dev/null +++ b/scripts/audit_release_boundary.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Fail when a release tracks private environments, upstream sources, or raw inputs.""" + +from __future__ import annotations + +import json +import re +import subprocess +from pathlib import Path + +FORBIDDEN_PARTS = {".venv", "source", "vendor", "__pycache__"} +SECRET_PATTERN = re.compile( + rb"(?:sk-proj-[A-Za-z0-9_-]{20,}|sk-ant-[A-Za-z0-9_-]{20,}|" + rb"(?:OPENAI|ANTHROPIC)_API_KEY\s*=\s*[^\s]+)" +) + + +def main() -> int: + root = Path(__file__).resolve().parents[1] + result = subprocess.run(["git", "ls-files", "-z"], cwd=root, capture_output=True, check=True) + tracked = [Path(item.decode("utf-8")) for item in result.stdout.split(b"\0") if item] + violations: list[str] = [] + for relative in tracked: + parts = set(relative.parts) + if FORBIDDEN_PARTS & parts: + violations.append(f"forbidden tracked directory: {relative}") + if relative.parts[:2] == ("data", "raw") and relative.name != ".gitkeep": + violations.append(f"raw input tracked: {relative}") + path = root / relative + if path.is_file() and path.stat().st_size <= 5_000_000: + try: + payload = path.read_bytes() + except OSError as error: + violations.append(f"unreadable tracked file: {relative}: {error}") + continue + if SECRET_PATTERN.search(payload): + violations.append(f"possible credential in tracked file: {relative}") + report = { + "status": "pass" if not violations else "fail", + "tracked_files": len(tracked), + "forbidden_parts": sorted(FORBIDDEN_PARTS), + "violations": violations, + } + print(json.dumps(report, indent=2, sort_keys=True)) + return 1 if violations else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_manuscript.py b/scripts/build_manuscript.py index 2e7e9b6..ff65379 100755 --- a/scripts/build_manuscript.py +++ b/scripts/build_manuscript.py @@ -79,6 +79,9 @@ def build_results(root: Path) -> tuple[str, str]: zhao_filtered = load_json(root / "data/zhao_vaccine_leakage_filter_summary.json") zhao_overlap = load_json(root / "research/training_overlap_summary_zhao.json") zhao_result = load_json(root / "results/analysis/zhao/fixed/metrics.json") + rcc_source = load_json(root / "data/rcc_vaccine_summary.json") + rcc_overlap = load_json(root / "research/training_overlap_summary_rcc.json") + rcc_result = load_json(root / "results/analysis/rcc/metrics.json") expanded_result = load_json(root / "results/analysis/improve/expanded_9_10/metrics.json") peptide_result = load_json(root / "results/analysis/improve/peptide_sensitivity/metrics.json") peptide_hla_rank_result = load_json( @@ -92,6 +95,7 @@ def build_results(root: Path) -> tuple[str, str]: fixed = load_csv(root / "results/tables/fixed_predictor_summary.csv") baselines = load_csv(root / "results/tables/heldout_baseline_summary.csv") hla = {row["predictor"]: row for row in load_csv(root / "results/analysis/improve/hla_sensitivity.csv")} + registry = load_csv(root / "data/predictor_registry.csv") benchmark_groups: dict[str, list[dict[str, str]]] = defaultdict(list) for row in load_csv(root / "data/processed/improve_benchmark.csv"): benchmark_groups[row["patient_id"]].append(row) @@ -121,8 +125,10 @@ def build_results(root: Path) -> tuple[str, str]: "baselines outperformed HLA-only baselines under both patient- and study-held-out fitting, " "while adding HLA to peptide features did not consistently improve over peptide features alone. " f"A frozen extension evaluated five models on {zhao_filtered['retained_rows']:,} overlap-filtered " - "vaccine peptides with a distinct post-vaccination ELISPOT endpoint. Support-matched random " - "ranking showed that high marginal Top-K values did not necessarily imply useful ranking signal." + "vaccine peptides with a distinct post-vaccination ELISPOT endpoint. A second endpoint-distinct " + f"vaccine cohort contributed {rcc_source['rows']} individually assayed short peptides from " + f"{rcc_source['patients']} patients. Support-matched and cross-domain analyses showed that high " + "marginal Top-K values did not necessarily imply stable or useful ranking signal." ) lopo = {row["predictor"]: row for row in baselines if row["analysis"] == "lopo"} @@ -154,6 +160,20 @@ def build_results(root: Path) -> tuple[str, str]: for row in zhao_result["paired_same_task"] if row["left"] == "BigMHC" and row["right"] == "PRIME" and row["metric"] == "ndcg@5" ) + rcc_rows = [] + rcc_benchmark = load_csv(root / "data/processed/rcc_vaccine_benchmark.csv") + for name, value in sorted(rcc_result["metrics"].items()): + random_ndcg5, _ = random_ranking_reference( + rcc_benchmark, load_csv(root / value["metadata"]["source"]), k=5 + ) + ci = value["patient_bootstrap_95ci"]["ndcg@5"] + rcc_rows.append( + f"| {name} | {value['pooled']['n']:,} | {f(value['pooled']['auroc'])} | " + f"{f(value['pooled']['average_precision'])} | {f(value['patient']['ndcg@5'])} " + f"({f(ci['low'])}–{f(ci['high'])}) | {f(random_ndcg5)} | " + f"{f(value['patient']['ndcg@5'] - random_ndcg5)} |" + ) + profile_only = sum(row["final_status"] != "reproduced" for row in registry) results = f"""### Public-artifact reproduction The version-pinned CPU workflows for MHCflurry 2.2.1, BigMHC v1.0 and PRIME 2.0 all produced complete outputs for the common benchmark. Reproduction nevertheless required tool-specific workarounds: MHCflurry model-path correction, a 4.6-GB BigMHC repository checkout and native rebuilding of PRIME and MixMHCpred binaries on Apple Silicon. These observations are recorded in `data/predictor_registry.csv`; they describe this platform and these pinned revisions rather than a universal installation-success rate. @@ -199,6 +219,22 @@ def build_results(root: Path) -> tuple[str, str]: | Predictor | Predicted records | AUROC | AP | Patient NDCG@5 (95% CI) | Random NDCG@5 | Gain over random | |---|---:|---:|---:|---:|---:|---:| {chr(10).join(zhao_rows)} + +### Endpoint-distinct RCC vaccine cohort + +The separately frozen RCC protocol retained {rcc_source['rows']} individually assayed short peptides from {rcc_source['patients']} vaccinated patients after excluding one source row with no usable short peptide/HLA assignment [@braun2025rcc]. The assay compared three peptide-stimulation replicates with three matched no-stimulation replicates; labels follow the source p-value threshold and therefore are assay-context outcomes rather than untreated biological negatives. No exact PRIME2, BigMHC-construction or DeepImmuno training overlap was identified among the {rcc_overlap['benchmark_rows']} records, while DeepHLApan row-level training identity remains unknown. + +On near-complete support, PRIME had AUROC {f(rcc_result['metrics']['PRIME']['pooled']['auroc'])} and patient NDCG@5 {f(rcc_result['metrics']['PRIME']['patient']['ndcg@5'])}; BigMHC had {f(rcc_result['metrics']['BigMHC']['pooled']['auroc'])} and {f(rcc_result['metrics']['BigMHC']['patient']['ndcg@5'])}, respectively. DeepImmuno-CNN supported only {rcc_result['metrics']['DeepImmuno-CNN']['pooled']['n']} records. With nine patients, all estimates are descriptive and do not establish a cross-domain interaction, universal ordering, natural tumour presentation or clinical efficacy. + +**Table 4. RCC personalized-vaccine cohort.** The endpoint is post-vaccination, individual-peptide IFN-γ ELISpot after in-vitro stimulation. Random NDCG@5 is calculated on each model's exact support. + +| Predictor | Predicted records | AUROC | AP | Patient NDCG@5 (95% CI) | Random NDCG@5 | Gain over random | +|---|---:|---:|---:|---:|---:|---:| +{chr(10).join(rcc_rows)} + +### Expanded reproducibility profile and extension contract + +The artifact census now records {len(registry)} pinned predictor entries. Beyond the five benchmarked tools, {profile_only} entries are retained as profile-only, non-comparable, pending or unreproducible outcomes rather than being silently omitted. The public extension contract supplies machine-validated Dataset Cards, Predictor Cards and prediction-artifact schemas together with a common-support evaluator; these additions improve reuse but do not make heterogeneous prediction tasks scientifically interchangeable. """ return abstract, results diff --git a/scripts/build_rcc_benchmark.py b/scripts/build_rcc_benchmark.py new file mode 100755 index 0000000..e1b608b --- /dev/null +++ b/scripts/build_rcc_benchmark.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +"""Build the RCC vaccine cohort from the checksum-pinned Nature workbook.""" + +import argparse +import csv +import hashlib +import json +import re +import xml.etree.ElementTree as ET +import zipfile +from pathlib import Path + +EXPECTED = "c113c42b0773049fe7e3f6b983485d15cd00cb847c6fd5de532cea4c9715d0c1" +AA = set("ACDEFGHIKLMNPQRSTVWY") +NS = {"m": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} + + +def text(v): + return "" if v is None else str(v).strip() + + +def norm_hla(v): + out = [] + for allele in re.split(r"[,;]", text(v)): + x = re.sub(r"[^A-Z0-9]", "", allele.upper().replace("HLA", "")) + m = re.fullmatch(r"([ABC])(\d{2})(\d{2})", x) + if not m: + raise ValueError(f"unsupported HLA {allele!r}") + out.append(f"HLA-{m.group(1)}*{m.group(2)}:{m.group(3)}") + return ";".join(out) + + +def rows(path): + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != EXPECTED: + raise ValueError("RCC member checksum mismatch") + z = zipfile.ZipFile(path) + ss = [] + root = ET.fromstring(z.read("xl/sharedStrings.xml")) + for si in root.findall("m:si", NS): + ss.append("".join(t.text or "" for t in si.findall(".//m:t", NS))) + root = ET.fromstring(z.read("xl/worksheets/sheet2.xml")) + table = [] + for ri, row in enumerate(root.findall(".//m:row", NS)): + vals = [] + for c in row.findall("m:c", NS): + v = c.find("m:v", NS) + val = "" if v is None else v.text + if c.get("t") == "s" and val: + val = ss[int(val)] + vals.append(val) + if ri == 0: + headers = vals + continue + d = dict(zip(headers, vals)) + pep = text(d["Short_Epitope"]).upper() + if pep.startswith("N/A") or text(d["HLA_of_best_short_epitope"]).upper() == "N/A": + continue + hla = norm_hla(d["HLA_of_best_short_epitope"]) + if not pep or set(pep) - AA or not 8 <= len(pep) <= 11: + raise ValueError(f"row {ri}: invalid peptide") + stim = [text(d[f"InVitro_PeptideStim_Replicate0{i}"]) for i in range(1, 4)] + nostim = [text(d[f"InVitro_NoStim_Replicate0{i}"]) for i in range(1, 4)] + if any(not x for x in stim + nostim): + raise ValueError(f"row {ri}: incomplete assay") + p = float(text(d["Ttest_pvalue_InVitroStim"])) + record = f"rcc-vaccine-{hashlib.sha256((text(d['Patient_ID']) + '|' + text(d['Peptide_ID']) + '|' + pep + '|' + hla).encode()).hexdigest()[:16]}" + table.append( + { + "record_id": record, + "patient_id": "RCC-" + text(d["Patient_ID"]), + "study_id": "RCC_PCV_VACCINE_2025", + "hla": hla, + "mhc_class": "I", + "peptide": pep, + "peptide_length": str(len(pep)), + "immunogenicity": str(int(p < 0.05)), + "label": str(int(p < 0.05)), + "assay_type": "post_vaccine_invitro_IFNG_ELISPOT", + "clinical_context": "personalized_RCC_peptide_vaccine", + "source_doi": "10.1038/s41586-024-08507-5", + "source_url": "https://doi.org/10.1038/s41586-024-08507-5", + "source_checksum": EXPECTED, + "source_row": str(ri + 1), + "source_pvalue": text(d["Ttest_pvalue_InVitroStim"]), + "evidence_level": "individual_peptide_stimulation_with_matched_no_stimulation_control", + } + ) + return table + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--input", type=Path, required=True) + ap.add_argument("--output", type=Path, required=True) + ap.add_argument("--summary", type=Path, required=True) + a = ap.parse_args() + data = rows(a.input) + assert len(data) == 129 and len({r["patient_id"] for r in data}) == 9 + a.output.parent.mkdir(parents=True, exist_ok=True) + a.summary.parent.mkdir(parents=True, exist_ok=True) + with a.output.open("w", newline="", encoding="utf-8") as f: + w = csv.DictWriter(f, fieldnames=list(data[0]), lineterminator="\n") + w.writeheader() + w.writerows(data) + summary = { + "rows": len(data), + "patients": len({r["patient_id"] for r in data}), + "positives": sum(int(r["label"]) for r in data), + "negatives": sum(not int(r["label"]) for r in data), + "member_sha256": EXPECTED, + "endpoint": "post-vaccine individual-peptide IFNG ELISpot", + "hla_semantics": "predicted best short-epitope binding allele", + } + a.summary.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(json.dumps(summary, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_results_manifest.py b/scripts/build_results_manifest.py index e1c4992..812f3c0 100755 --- a/scripts/build_results_manifest.py +++ b/scripts/build_results_manifest.py @@ -86,6 +86,41 @@ "reports/improve_expanded_predictor_run.json", "reports/new_predictor_determinism.json", "reports/extension_clean_reproduction.json", + "data/processed/rcc_vaccine_benchmark.csv", + "data/processed/rcc_vaccine_benchmark_full.csv", + "data/rcc_vaccine_summary.json", + "research/extension_protocol_rcc_v1.json", + "research/training_overlap_audit_rcc.csv", + "research/training_overlap_summary_rcc.json", + "results/raw_predictions/rcc/bigmhc-v1.0.csv", + "results/raw_predictions/rcc/prime-2.0.csv", + "results/raw_predictions/rcc/deepimmuno-cnn.csv", + "results/raw_predictions/rcc/deephlapan-1.1.1.csv", + "reports/rcc_predictor_run.json", + "results/analysis/rcc/metrics.json", + "results/analysis/rcc/missingness.csv", + "results/analysis/rcc/paired_differences.csv", + "results/analysis/stability/analysis_metadata.json", + "results/analysis/stability/dataset_predictor_metric_matrix.csv", + "results/analysis/stability/rank_stability.csv", + "results/analysis/stability/model_selection_first_probability.csv", + "results/analysis/stability/sensitivity_summary.csv", + "results/analysis/stability/leave_one_domain_out.csv", + "results/analysis/stability/endpoint_domain_metadata.csv", + "results/analysis/stability/endpoint_domain_auroc.svg", + "results/analysis/stability/endpoint_domain_auroc.png", + "contracts/dataset-card.example.json", + "contracts/predictor-card.example.json", + "contracts/prediction-artifact.schema.json", + "contracts/synthetic/benchmark.csv", + "contracts/synthetic/predictions.csv", + "predictors/mhcmatch/attempts/sweep_receipt.json", + "predictors/mhcnuggets/attempts/sweep_receipt.json", + "predictors/neofox/attempts/sweep_receipt.json", + "predictors/neoguider/attempts/sweep_receipt.json", + "predictors/pvactools/attempts/sweep_receipt.json", + "predictors/seq2neo/attempts/sweep_receipt.json", + "predictors/vaxrank/attempts/sweep_receipt.json", ) diff --git a/scripts/evaluate_benchmark.py b/scripts/evaluate_benchmark.py index 40b1092..c0ab2ae 100755 --- a/scripts/evaluate_benchmark.py +++ b/scripts/evaluate_benchmark.py @@ -4,7 +4,9 @@ from __future__ import annotations import argparse +import ast import csv +import hashlib import json import math import random @@ -181,6 +183,26 @@ def load_csv(path: Path) -> list[dict[str, str]]: return list(csv.DictReader(handle)) +def input_fingerprint(paths: list[Path], bootstrap: int, seed: int) -> str: + """Bind a resumable bootstrap checkpoint to its exact inputs and configuration.""" + digest = hashlib.sha256() + digest.update(f"bootstrap={bootstrap}\nseed={seed}\n".encode()) + for path in paths: + digest.update(str(path).encode()) + digest.update(b"\0") + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def atomic_json(path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, sort_keys=True, allow_nan=False) + "\n") + temporary.replace(path) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--benchmark", type=Path, required=True) @@ -188,6 +210,12 @@ def main() -> int: parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--bootstrap", type=int, default=2000) parser.add_argument("--seed", type=int, default=20260820) + parser.add_argument( + "--checkpoint-every", + type=int, + default=50, + help="atomically save bootstrap state every N replicates; 0 disables resume", + ) args = parser.parse_args() benchmark_rows = load_csv(args.benchmark) @@ -333,7 +361,32 @@ def main() -> int: } for predictor, rows in joined.items() } - for _ in range(args.bootstrap): + checkpoint_path = args.output_dir / ".bootstrap_checkpoint.json" + fingerprint = input_fingerprint( + [args.benchmark, *sorted(args.predictions)], args.bootstrap, args.seed + ) + start_replicate = 0 + if args.checkpoint_every > 0 and checkpoint_path.exists(): + checkpoint = json.loads(checkpoint_path.read_text()) + if checkpoint.get("fingerprint") != fingerprint: + raise SystemExit( + f"checkpoint inputs/configuration changed; remove {checkpoint_path} to restart" + ) + start_replicate = int(checkpoint["completed"]) + rng.setstate(ast.literal_eval(checkpoint["rng_state"])) + samples = { + predictor: defaultdict(list, metric_samples) + for predictor, metric_samples in checkpoint["samples"].items() + } + restored_pairs = {} + for item in checkpoint["paired_samples"]: + restored_pairs[(item["left"], item["right"])] = defaultdict( + list, item["metrics"] + ) + paired_samples = restored_pairs + print(f"resuming bootstrap at replicate {start_replicate}/{args.bootstrap}") + + for replicate_index in range(start_replicate, args.bootstrap): draws = rng.choices(patients, k=len(patients)) replicate = {} for predictor in joined: @@ -383,6 +436,26 @@ def main() -> int: right_value = pair_replicate[right][metric] if isinstance(left_value, (int, float)) and isinstance(right_value, (int, float)): paired_samples[pair][metric].append(left_value - right_value) + completed = replicate_index + 1 + if ( + args.checkpoint_every > 0 + and completed < args.bootstrap + and completed % args.checkpoint_every == 0 + ): + atomic_json( + checkpoint_path, + { + "schema_version": 1, + "fingerprint": fingerprint, + "completed": completed, + "rng_state": repr(rng.getstate()), + "samples": samples, + "paired_samples": [ + {"left": pair[0], "right": pair[1], "metrics": values} + for pair, values in sorted(paired_samples.items()) + ], + }, + ) for predictor, predictor_metrics in metrics.items(): predictor_metrics["patient_bootstrap_95ci"] = { @@ -426,7 +499,9 @@ def main() -> int: json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n" ) with (args.output_dir / "missingness.csv").open("w", newline="") as handle: - writer = csv.DictWriter(handle, fieldnames=["predictor", "status", "count"]) + writer = csv.DictWriter( + handle, fieldnames=["predictor", "status", "count"], lineterminator="\n" + ) writer.writeheader() writer.writerows(missingness) with (args.output_dir / "paired_differences.csv").open("w", newline="") as handle: @@ -444,9 +519,10 @@ def main() -> int: "ci_low", "ci_high", ] - writer = csv.DictWriter(handle, fieldnames=fields) + writer = csv.DictWriter(handle, fieldnames=fields, lineterminator="\n") writer.writeheader() writer.writerows(paired) + checkpoint_path.unlink(missing_ok=True) print( json.dumps( { diff --git a/scripts/render_stability_figure.py b/scripts/render_stability_figure.py new file mode 100755 index 0000000..e9fa23f --- /dev/null +++ b/scripts/render_stability_figure.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Render a dependency-free SVG from the stability matrix.""" +import csv +import html +from pathlib import Path + +root=Path(__file__).resolve().parents[1]; out=root/"results/analysis/stability"; rows=list(csv.DictReader((out/"dataset_predictor_metric_matrix.csv").open())) +ds=sorted({r["dataset"] for r in rows}); names=sorted({r["predictor"] for r in rows}); W,H=900,100+len(names)*34; left,top,cell=220,55,110 +def value(d,n): + for r in rows: + if r["dataset"]==d and r["predictor"]==n and r["metric"]=="AUROC": return float(r["value"]) + return None +def color(v): + if v is None:return "#d9d9d9" + q=max(0,min(1,v)); return f"rgb({int(245-180*q)},{int(245-120*q)},{int(245-20*q)})" +s=[f'', '', 'Exploratory AUROC by endpoint/domain and predictor'] +for j,d in enumerate(ds): s += [f'{html.escape(d)}'] +for i,n in enumerate(names): + y=top+i*34; s.append(f'{html.escape(n)}') + for j,d in enumerate(ds): + v=value(d,n); x=left+j*cell; s.append(f''); s.append(f'{"NA" if v is None else f"{v:.3f}"}') +s += [f'Fixed pretrained scores; descriptive heterogeneity only; endpoint/domain differences are not causal estimates.',''] +(out/"endpoint_domain_auroc.svg").write_text("\n".join(s)+"\n") diff --git a/scripts/reproduce_public_predictors.py b/scripts/reproduce_public_predictors.py new file mode 100755 index 0000000..350ad8f --- /dev/null +++ b/scripts/reproduce_public_predictors.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +"""Cheap, auditable first-pass reproduction sweep for public predictors. + +This deliberately stops at installation/import/CLI smoke tests. It never +silently converts an end-to-end workflow into a peptide-HLA benchmark model. +""" + +from __future__ import annotations + +import argparse +import json +import platform +import shutil +import subprocess +import time +from pathlib import Path + +CANDIDATES = { + "mhcnuggets": "https://github.com/KarchinLab/mhcnuggets.git", + "neoguider": "https://github.com/XuegongLab/neoguider.git", + "neofox": "https://github.com/TRON-Bioinformatics/neofox.git", + "pvactools": "https://github.com/griffithlab/pVACtools.git", + "seq2neo": "https://github.com/XSLiuLab/Seq2Neo.git", + "vaxrank": "https://github.com/openvax/vaxrank.git", + "mhcmatch": "https://github.com/antigenomics/mhcmatch.git", +} +IMPORTS = {"mhcnuggets": "mhcnuggets", "neofox": "neofox", "vaxrank": "vaxrank"} +CLI = {"pvactools": "pvacseq", "vaxrank": "vaxrank"} + + +def run(cmd: list[str], cwd: Path | None = None, timeout: int = 180): + t = time.monotonic() + try: + p = subprocess.run( + cmd, cwd=cwd, text=True, capture_output=True, timeout=timeout, check=False + ) + return p.returncode, p.stdout, p.stderr, time.monotonic() - t + except subprocess.TimeoutExpired as e: + return ( + 124, + e.stdout or "", + (e.stderr or "") + f"\nTimed out after {timeout}s", + time.monotonic() - t, + ) + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--only", nargs="*", choices=sorted(CANDIDATES), default=sorted(CANDIDATES)) + args = ap.parse_args() + root = Path(__file__).resolve().parents[1] + uv = shutil.which("uv") + rows = [] + for name in args.only: + base = root / "predictors" / name + source = base / "source" + attempts = base / "attempts" + attempts.mkdir(parents=True, exist_ok=True) + if not source.exists(): + rc, out, err, _sec = run( + ["git", "clone", "--filter=blob:none", CANDIDATES[name], str(source)], root, 300 + ) + (attempts / "clone.stdout.log").write_text(out) + (attempts / "clone.stderr.log").write_text(err) + rc, rev, err, _sec = run(["git", "rev-parse", "HEAD"], source) + revision = rev.strip() if rc == 0 else "unknown" + license_files = sorted( + str(p.relative_to(source)) + for p in source.rglob("*") + if p.is_file() + and ("license" in p.name.lower() or p.name.lower() in {"copying", "notice"}) + )[:20] + env = base / ".venv" + install_rc = 125 + install_out = install_err = "not attempted" + if uv and source.exists(): + if not env.exists(): + run([uv, "venv", "--python", "3.11", str(env)], root, 180) + py = str(env / "bin/python") + install_rc, install_out, install_err, _ = run( + [uv, "pip", "install", "--python", py, "-e", str(source)], root, 900 + ) + (attempts / "install.stdout.log").write_text(install_out or "") + (attempts / "install.stderr.log").write_text(install_err or "") + smoke_rc = 125 + smoke_out = smoke_err = "not attempted" + if install_rc == 0 and name in IMPORTS: + smoke_rc, smoke_out, smoke_err, _ = run( + [str(env / "bin/python"), "-c", f"import {IMPORTS[name]}; print('import ok')"], + root, + 180, + ) + elif install_rc == 0 and name in CLI: + smoke_rc, smoke_out, smoke_err, _ = run( + [str(env / ("bin/" + CLI[name])), "--help"], root, 180 + ) + (attempts / "smoke.stdout.log").write_text(smoke_out or "") + (attempts / "smoke.stderr.log").write_text(smoke_err or "") + receipt = { + "predictor": name, + "revision": revision, + "repo": CANDIDATES[name], + "install_returncode": install_rc, + "smoke_returncode": smoke_rc, + "license_files": license_files, + "platform": platform.platform(), + "benchmark_track": "profile_only", + } + (attempts / "sweep_receipt.json").write_text(json.dumps(receipt, indent=2) + "\n") + rows.append((name, revision, install_rc, smoke_rc, ";".join(license_files))) + print("predictor\trevision\tinstall_rc\tsmoke_rc\tlicense_files") + for row in rows: + print("\t".join(map(str, row))) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_external_cohort_funnel.py b/scripts/validate_external_cohort_funnel.py new file mode 100755 index 0000000..e907eba --- /dev/null +++ b/scripts/validate_external_cohort_funnel.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Deterministic validation for the external-cohort screening funnel.""" + +import csv +import json +import pathlib + +root = pathlib.Path(__file__).resolve().parents[1] +csv_path = root / "research" / "external_cohort_funnel.csv" +json_path = root / "research" / "external_cohort_failure_protocols.json" +allowed = {"eligible", "pending", "failed"} +required = {"candidate_id", "eligibility", "decision", "evidence_note", "source_url"} +with csv_path.open(newline="", encoding="utf-8") as fh: + reader = csv.DictReader(fh) + assert reader.fieldnames is not None + rows = list(reader) +assert all(None not in row for row in rows), "malformed CSV row" +assert rows and required <= set(rows[0]), "missing funnel columns" +ids = {r["candidate_id"] for r in rows} +assert len(ids) == len(rows), "duplicate candidate_id" +for row in rows: + assert row["eligibility"] in allowed, row + assert row["decision"] in { + "existing_primary", + "existing_external", + "eligible_external", + "failed_eligibility", + "failed_leakage_gate", + "pending", + "failed", + }, row + assert row["source_url"].startswith(("http://", "https://")), row +data = json.loads(json_path.read_text(encoding="utf-8")) +failure_ids = {x["candidate_id"] for x in data["failures"]} +assert failure_ids <= ids +pending_ids = {x["candidate_id"] for x in data["pending"]} +assert pending_ids <= ids +assert not failure_ids & pending_ids +eligible_ids = {x["candidate_id"] for x in data["eligible_after_member_audit"]} +assert eligible_ids == { + r["candidate_id"] + for r in rows + if r["eligibility"] == "eligible" and r["decision"] == "eligible_external" +} +print(f"validated {len(rows)} candidates: {len(failure_ids)} failures, {len(pending_ids)} pending") diff --git a/src/neorepro/audit.py b/src/neorepro/audit.py new file mode 100644 index 0000000..30f70b9 --- /dev/null +++ b/src/neorepro/audit.py @@ -0,0 +1,475 @@ +"""Audit externally generated predictor scores against frozen NeoRepro benchmarks.""" + +from __future__ import annotations + +import csv +import math +import random +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from statistics import mean + +from neorepro.metrics import tie_aware_ranking_metrics + +INPUT_COLUMNS = ("patient_id", "peptide", "hla", "score", "model") +KS = (5, 20) +CORE_METRICS = ("recall@5", "recall@20", "ndcg@5", "mrr") +BOOTSTRAP_REPLICATES = 1000 +BOOTSTRAP_SEED = 20260820 + +Key = tuple[str, str, str] + + +class AuditError(ValueError): + """Raised when an audit cannot be performed without guessing.""" + + +@dataclass(frozen=True) +class BenchmarkSpec: + name: str + benchmark_path: str + overlap_path: str + + +@dataclass(frozen=True) +class BenchmarkRecord: + dataset: str + patient_id: str + peptide: str + hla: str + label: int + record_ids: tuple[str, ...] + + +@dataclass(frozen=True) +class Catalog: + records: dict[Key, BenchmarkRecord] + dataset_keys: dict[str, frozenset[Key]] + overlap_rows: dict[str, dict[str, dict[str, str]]] + collapsed_duplicate_rows: dict[str, int] + + +SPECS = ( + BenchmarkSpec( + "TESLA", + "data/processed/benchmark.csv", + "research/training_overlap_audit.csv", + ), + BenchmarkSpec( + "IMPROVE", + "data/processed/improve_benchmark_full.csv", + "research/training_overlap_audit_improve.csv", + ), + BenchmarkSpec( + "Zhao2026", + "data/processed/zhao_vaccine_benchmark_full.csv", + "research/training_overlap_audit_zhao.csv", + ), +) + +MODEL_ALIASES = { + "prime": "prime2", + "prime2": "prime2", + "prime20": "prime2", + "bigmhc": "bigmhc", + "bigmhcv10": "bigmhc", + "deepimmuno": "deepimmuno", + "deepimmunocnn": "deepimmuno", + "deephlapan": "deephlapan", + "deephlapan111": "deephlapan", +} + +OVERLAP_FIELDS: dict[str, dict[str, dict[str, str | None]]] = { + "TESLA": { + "prime2": { + "exact": "exact_peptide_hla_in_prime2_train", + "peptide_only": "peptide_in_prime2_train", + "near": None, + }, + "bigmhc": { + "exact": "exact_bigmhc_im_trainval_overlap", + "peptide_only": None, + "near": None, + }, + }, + "IMPROVE": { + "prime2": { + "exact": "exact_peptide_hla_in_prime2_train", + "peptide_only": "peptide_in_prime2_train", + "near": "near_hamming1_same_hla_prime2_train", + }, + "bigmhc": { + "exact": "exact_bigmhc_im_trainval_overlap", + "peptide_only": None, + "near": None, + }, + }, + "Zhao2026": { + "prime2": { + "exact": "exact_prime2_peptide_hla", + "peptide_only": "peptide_only_prime2_different_hla", + "near": "near_hamming1_prime2_same_hla", + }, + "bigmhc": { + "exact": "exact_bigmhc_im_trainval", + "peptide_only": None, + "near": None, + }, + "deepimmuno": { + "exact": "exact_deepimmuno_peptide_hla", + "peptide_only": "peptide_only_deepimmuno_different_hla", + "near": "near_hamming1_deepimmuno_same_hla", + }, + }, +} + + +def _read_csv(path: Path) -> tuple[list[str], list[dict[str, str]]]: + if not path.exists(): + raise AuditError(f"missing required frozen artifact: {path}") + with path.open(newline="", encoding="utf-8-sig") as handle: + reader = csv.DictReader(handle) + if reader.fieldnames is None: + raise AuditError(f"CSV has no header: {path}") + return reader.fieldnames, list(reader) + + +def _key(patient_id: str, peptide: str, hla: str) -> Key: + return patient_id.strip(), peptide.strip().upper(), hla.strip() + + +def load_catalog(root: Path) -> Catalog: + grouped: dict[Key, list[tuple[str, dict[str, str]]]] = defaultdict(list) + overlap_rows: dict[str, dict[str, dict[str, str]]] = {} + for spec in SPECS: + _header, rows = _read_csv(root / spec.benchmark_path) + for row in rows: + grouped[_key(row["patient_id"], row["peptide"], row["hla"])].append( + (spec.name, row) + ) + _overlap_header, audit_rows = _read_csv(root / spec.overlap_path) + overlap_by_id = {row["record_id"]: row for row in audit_rows} + if len(overlap_by_id) != len(audit_rows): + raise AuditError(f"duplicate record_id in {spec.overlap_path}") + overlap_rows[spec.name] = overlap_by_id + + records: dict[Key, BenchmarkRecord] = {} + dataset_keys: dict[str, set[Key]] = defaultdict(set) + collapsed: dict[str, int] = defaultdict(int) + for key, members in grouped.items(): + datasets = {dataset for dataset, _row in members} + if len(datasets) != 1: + raise AuditError(f"ambiguous cross-dataset benchmark key: {key}") + dataset = next(iter(datasets)) + labels = {row["immunogenicity"] for _dataset, row in members} + if not labels <= {"0", "1"} or len(labels) != 1: + raise AuditError(f"conflicting or unknown labels for benchmark key: {key}") + record_ids = tuple(sorted(row["record_id"] for _dataset, row in members)) + if any(record_id not in overlap_rows[dataset] for record_id in record_ids): + raise AuditError(f"missing training-overlap row for benchmark key: {key}") + records[key] = BenchmarkRecord( + dataset=dataset, + patient_id=key[0], + peptide=key[1], + hla=key[2], + label=int(next(iter(labels))), + record_ids=record_ids, + ) + dataset_keys[dataset].add(key) + collapsed[dataset] += len(members) - 1 + return Catalog( + records=records, + dataset_keys={name: frozenset(keys) for name, keys in dataset_keys.items()}, + overlap_rows=overlap_rows, + collapsed_duplicate_rows=dict(collapsed), + ) + + +def _model_family(model: str) -> str | None: + normalized = re.sub(r"[^a-z0-9]+", "", model.lower()) + return MODEL_ALIASES.get(normalized) + + +def _truthy_overlap(value: str) -> bool: + return value.strip() == "1" + + +def _record_has_overlap( + record: BenchmarkRecord, + overlap_rows: dict[str, dict[str, dict[str, str]]], + field: str, +) -> bool: + return any( + _truthy_overlap(overlap_rows[record.dataset][record_id].get(field, "")) + for record_id in record.record_ids + ) + + +def _leakage_for_model( + model: str, + supported: set[Key], + datasets: set[str], + catalog: Catalog, +) -> tuple[dict[str, object], set[Key]]: + family = _model_family(model) + if family is None: + return ( + { + "risk": "unknown_training_reference", + "recognized_training_reference": False, + "exact_overlap_records": None, + "peptide_only_overlap_records": None, + "near_overlap_records": None, + "checked_datasets": [], + "unknown_datasets": sorted(datasets), + }, + set(), + ) + + exact_keys: set[Key] = set() + peptide_only_keys: set[Key] = set() + near_keys: set[Key] = set() + exact_checked: set[str] = set() + peptide_checked: set[str] = set() + near_checked: set[str] = set() + unknown_datasets: set[str] = set() + for dataset in datasets: + fields = OVERLAP_FIELDS.get(dataset, {}).get(family) + if fields is None or fields["exact"] is None: + unknown_datasets.add(dataset) + continue + exact_checked.add(dataset) + if fields["peptide_only"] is not None: + peptide_checked.add(dataset) + if fields["near"] is not None: + near_checked.add(dataset) + for key in supported & set(catalog.dataset_keys[dataset]): + record = catalog.records[key] + has_exact_overlap = _record_has_overlap( + record, catalog.overlap_rows, str(fields["exact"]) + ) + if has_exact_overlap: + exact_keys.add(key) + peptide_field = fields["peptide_only"] + if ( + peptide_field + and not has_exact_overlap + and _record_has_overlap(record, catalog.overlap_rows, peptide_field) + ): + peptide_only_keys.add(key) + near_field = fields["near"] + if near_field and _record_has_overlap(record, catalog.overlap_rows, near_field): + near_keys.add(key) + + if exact_keys: + risk = "high_exact_overlap" + elif near_keys or peptide_only_keys: + risk = "possible_sequence_overlap" + elif unknown_datasets: + risk = "partially_unknown" + else: + risk = "no_known_overlap_in_checked_dimensions" + return ( + { + "risk": risk, + "recognized_training_reference": True, + "model_family": family, + "exact_overlap_records": len(exact_keys) if exact_checked else None, + "peptide_only_overlap_records": ( + len(peptide_only_keys) if peptide_checked else None + ), + "near_overlap_records": len(near_keys) if near_checked else None, + "checked_datasets": sorted(exact_checked), + "unknown_datasets": sorted(unknown_datasets), + "excluded_from_metrics_as_exact_overlap": len(exact_keys), + }, + exact_keys, + ) + + +def _percentile(values: list[float], probability: float) -> float: + ordered = sorted(values) + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _patient_metric_report( + keys: set[Key], + scores: dict[Key, float | None], + catalog: Catalog, +) -> dict[str, object]: + if not keys: + return { + "status": "no_leakage_filtered_common_support", + "positive_bearing_patients": 0, + "metrics": {}, + } + by_patient: dict[str, list[Key]] = defaultdict(list) + for key in sorted(keys): + by_patient[catalog.records[key].patient_id].append(key) + patient_values: list[dict[str, float]] = [] + for patient_keys in by_patient.values(): + labels = [catalog.records[key].label for key in patient_keys] + if not sum(labels): + continue + numeric_scores = [scores[key] for key in patient_keys] + if any(score is None for score in numeric_scores): + raise AssertionError("common-support keys must have numeric scores") + patient_values.append( + tie_aware_ranking_metrics( + labels, + [float(score) for score in numeric_scores], + KS, + ) + ) + if not patient_values: + return { + "status": "no_positive_bearing_patients", + "positive_bearing_patients": 0, + "metrics": {}, + } + + rng = random.Random(BOOTSTRAP_SEED) + samples: dict[str, list[float]] = {metric: [] for metric in CORE_METRICS} + for _ in range(BOOTSTRAP_REPLICATES): + draw = rng.choices(patient_values, k=len(patient_values)) + for metric in CORE_METRICS: + samples[metric].append(mean(values[metric] for values in draw)) + metrics = {} + for metric in CORE_METRICS: + values = samples[metric] + metrics[metric] = { + "estimate": mean(patient[metric] for patient in patient_values), + "ci95": { + "low": _percentile(values, 0.025), + "high": _percentile(values, 0.975), + }, + } + return { + "status": "ok", + "positive_bearing_patients": len(patient_values), + "bootstrap_unit": "patient", + "bootstrap_replicates": BOOTSTRAP_REPLICATES, + "seed": BOOTSTRAP_SEED, + "metrics": metrics, + } + + +def audit_predictions(path: Path, root: Path) -> dict[str, object]: + """Return a deterministic audit report for a five-column prediction CSV.""" + header, rows = _read_csv(path) + if len(header) != len(INPUT_COLUMNS) or set(header) != set(INPUT_COLUMNS): + raise AuditError( + "prediction CSV header must contain exactly: " + ", ".join(INPUT_COLUMNS) + ) + if not rows: + raise AuditError("prediction CSV has no data rows") + + catalog = load_catalog(root) + predictions: dict[str, dict[Key, float | None]] = defaultdict(dict) + inferred_datasets: set[str] = set() + for line_number, row in enumerate(rows, start=2): + model = row["model"].strip() + key = _key(row["patient_id"], row["peptide"], row["hla"]) + if not model or not all(key): + raise AuditError(f"line {line_number}: patient_id, peptide, hla and model are required") + if key not in catalog.records: + raise AuditError(f"line {line_number}: key not found in a frozen benchmark: {key}") + if key in predictions[model]: + raise AuditError(f"line {line_number}: duplicate model/benchmark key for {model}: {key}") + score_text = row["score"].strip() + score = None + if score_text: + try: + score = float(score_text) + except ValueError as error: + raise AuditError(f"line {line_number}: score is not numeric") from error + if not math.isfinite(score): + raise AuditError(f"line {line_number}: score must be finite") + predictions[model][key] = score + inferred_datasets.add(catalog.records[key].dataset) + + expected_keys = set().union( + *(set(catalog.dataset_keys[dataset]) for dataset in inferred_datasets) + ) + models = sorted(predictions) + support_by_model: dict[str, set[Key]] = {} + support_report = {} + for model in models: + submitted = set(predictions[model]) + supported = {key for key, score in predictions[model].items() if score is not None} + support_by_model[model] = supported + support_report[model] = { + "expected_records": len(expected_keys), + "submitted_records": len(submitted), + "supported_records": len(supported), + "blank_score_records": len(submitted - supported), + "omitted_records": len(expected_keys - submitted), + "coverage": len(supported) / len(expected_keys), + } + + raw_common = set.intersection(*(support_by_model[model] for model in models)) + leakage_report = {} + exact_exclusions = {} + for model in models: + leakage_report[model], exact_exclusions[model] = _leakage_for_model( + model, + support_by_model[model], + inferred_datasets, + catalog, + ) + audited_support = { + model: support_by_model[model] - exact_exclusions[model] for model in models + } + audited_common = set.intersection(*(audited_support[model] for model in models)) + + patient_metrics = { + model: _patient_metric_report(audited_common, predictions[model], catalog) + for model in models + } + return { + "schema_version": 1, + "input": { + "path": str(path), + "columns": list(INPUT_COLUMNS), + "score_direction": "higher_is_better", + "rows": len(rows), + "models": models, + }, + "benchmark": { + "datasets": sorted(inferred_datasets), + "expected_unique_patient_peptide_hla_records": len(expected_keys), + "collapsed_duplicate_source_rows": sum( + catalog.collapsed_duplicate_rows.get(dataset, 0) + for dataset in inferred_datasets + ), + }, + "leakage": leakage_report, + "support": { + "by_model": support_report, + "raw_common_support": { + "records": len(raw_common), + "coverage": len(raw_common) / len(expected_keys), + }, + "leakage_filtered_common_support": { + "records": len(audited_common), + "coverage": len(audited_common) / len(expected_keys), + }, + }, + "patient_metrics": { + "basis": "leakage-filtered common support; pMHC ranking unit", + "models": patient_metrics, + }, + "limitations": [ + "Only exact model aliases with frozen training references receive checked leakage labels.", + "Unknown training overlap remains unknown; absence of a known match is not proof of independence.", + "Omitted input rows are counted as unsupported because the five-column contract has no status field.", + ], + } diff --git a/src/neorepro/benchmark.py b/src/neorepro/benchmark.py new file mode 100644 index 0000000..89f6762 --- /dev/null +++ b/src/neorepro/benchmark.py @@ -0,0 +1,615 @@ +"""Portable, dependency-free benchmark entry point for external predictions.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import math +import platform +import random +from collections import defaultdict +from dataclasses import dataclass +from itertools import combinations +from pathlib import Path +from statistics import mean + +from neorepro.metrics import auroc, average_precision, tie_aware_ranking_metrics + +REQUIRED_COLUMNS = ("record_id", "patient_id", "study_id", "label", "score", "predictor") +OPTIONAL_COLUMNS = ( + "score_direction", + "status", + "training_overlap", + "hla", + "assay", + "cancer_type", +) +VALID_DIRECTIONS = {"higher", "lower"} +VALID_STATUSES = {"predicted", "unsupported", "failed", "invalid"} +VALID_OVERLAP = {"exact", "none", "unknown"} +KS = (5, 10, 20) +PATIENT_METRICS = ("mrr",) + tuple( + f"{metric}@{k}" + for k in KS + for metric in ("recall", "precision", "hitrate", "ndcg") +) + + +class BenchmarkError(ValueError): + """Raised when a standard prediction submission is invalid.""" + + +@dataclass(frozen=True) +class Record: + record_id: str + patient_id: str + study_id: str + label: int + hla: str + assay: str + cancer_type: str + + +@dataclass(frozen=True) +class Submission: + path: Path + records: dict[str, Record] + scores: dict[str, dict[str, float | None]] + statuses: dict[str, dict[str, str]] + overlaps: dict[str, dict[str, str]] + directions: dict[str, str] + rows: int + columns: tuple[str, ...] + sha256: str + + +def _percentile(values: list[float], probability: float) -> float: + ordered = sorted(values) + position = probability * (len(ordered) - 1) + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def _interval(values: list[float]) -> dict[str, float] | None: + if not values: + return None + return {"low": _percentile(values, 0.025), "high": _percentile(values, 0.975)} + + +def _parse_label(value: str, line: int) -> int: + if value.strip() not in {"0", "1"}: + raise BenchmarkError(f"line {line}: label must be 0 or 1") + return int(value) + + +def read_submission(path: Path, default_direction: str = "higher") -> Submission: + """Read and strictly validate one portable benchmark CSV.""" + if default_direction not in VALID_DIRECTIONS: + raise BenchmarkError("default score direction must be higher or lower") + try: + raw = path.read_bytes() + except OSError as error: + raise BenchmarkError(f"cannot read {path}: {error}") from error + try: + text = raw.decode("utf-8-sig") + except UnicodeDecodeError as error: + raise BenchmarkError("prediction CSV must be UTF-8 encoded") from error + + reader = csv.DictReader(text.splitlines()) + if reader.fieldnames is None: + raise BenchmarkError("prediction CSV has no header") + columns = tuple(reader.fieldnames) + missing = [column for column in REQUIRED_COLUMNS if column not in columns] + if missing: + raise BenchmarkError("prediction CSV missing columns: " + ", ".join(missing)) + + records: dict[str, Record] = {} + optional_values: dict[str, dict[str, str]] = defaultdict(dict) + scores: dict[str, dict[str, float | None]] = defaultdict(dict) + statuses: dict[str, dict[str, str]] = defaultdict(dict) + overlaps: dict[str, dict[str, str]] = defaultdict(dict) + directions: dict[str, str] = {} + rows = 0 + for line, row in enumerate(reader, start=2): + rows += 1 + record_id = row["record_id"].strip() + patient_id = row["patient_id"].strip() + study_id = row["study_id"].strip() + predictor = row["predictor"].strip() + if not all((record_id, patient_id, study_id, predictor)): + raise BenchmarkError( + f"line {line}: record_id, patient_id, study_id and predictor are required" + ) + label = _parse_label(row["label"], line) + metadata = { + field: row.get(field, "").strip() for field in ("hla", "assay", "cancer_type") + } + if record_id in records: + existing = records[record_id] + if (existing.patient_id, existing.study_id, existing.label) != ( + patient_id, + study_id, + label, + ): + raise BenchmarkError(f"line {line}: conflicting truth metadata for {record_id}") + for field, value in metadata.items(): + previous = optional_values[record_id].get(field, "") + if previous and value and previous != value: + raise BenchmarkError( + f"line {line}: conflicting {field} metadata for {record_id}" + ) + if value: + optional_values[record_id][field] = value + else: + optional_values[record_id] = metadata + records[record_id] = Record( + record_id=record_id, + patient_id=patient_id, + study_id=study_id, + label=label, + hla=metadata["hla"], + assay=metadata["assay"], + cancer_type=metadata["cancer_type"], + ) + + if record_id in scores[predictor]: + raise BenchmarkError(f"line {line}: duplicate predictor/record_id pair") + direction = row.get("score_direction", "").strip() or default_direction + if direction not in VALID_DIRECTIONS: + raise BenchmarkError(f"line {line}: score_direction must be higher or lower") + if predictor in directions and directions[predictor] != direction: + raise BenchmarkError(f"line {line}: inconsistent score direction for {predictor}") + directions[predictor] = direction + + score_text = row["score"].strip() + status = row.get("status", "").strip() or ("predicted" if score_text else "unsupported") + if status not in VALID_STATUSES: + raise BenchmarkError(f"line {line}: invalid status {status!r}") + score: float | None = None + if status == "predicted": + try: + score = float(score_text) + except ValueError as error: + raise BenchmarkError(f"line {line}: predicted score must be numeric") from error + if not math.isfinite(score): + raise BenchmarkError(f"line {line}: predicted score must be finite") + elif score_text: + raise BenchmarkError(f"line {line}: non-predicted rows must have a blank score") + + overlap = row.get("training_overlap", "").strip() or "unknown" + if overlap not in VALID_OVERLAP: + raise BenchmarkError( + f"line {line}: training_overlap must be exact, none, or unknown" + ) + scores[predictor][record_id] = score + statuses[predictor][record_id] = status + overlaps[predictor][record_id] = overlap + + if not rows: + raise BenchmarkError("prediction CSV has no data rows") + + # Rebuild records after nonblank optional values have been reconciled across predictors. + records = { + record_id: Record( + record_id=record.record_id, + patient_id=record.patient_id, + study_id=record.study_id, + label=record.label, + hla=optional_values[record_id].get("hla", ""), + assay=optional_values[record_id].get("assay", ""), + cancer_type=optional_values[record_id].get("cancer_type", ""), + ) + for record_id, record in records.items() + } + return Submission( + path=path, + records=records, + scores=dict(scores), + statuses=dict(statuses), + overlaps=dict(overlaps), + directions=directions, + rows=rows, + columns=columns, + sha256=hashlib.sha256(raw).hexdigest(), + ) + + +def _oriented(score: float, direction: str) -> float: + return score if direction == "higher" else -score + + +def _classification_metrics( + labels: list[int], raw_scores: list[float], direction: str, threshold: float +) -> dict[str, object]: + oriented = [_oriented(score, direction) for score in raw_scores] + predicted = [ + int(score >= threshold) if direction == "higher" else int(score <= threshold) + for score in raw_scores + ] + tp = sum(label == 1 and call == 1 for label, call in zip(labels, predicted, strict=True)) + tn = sum(label == 0 and call == 0 for label, call in zip(labels, predicted, strict=True)) + fp = sum(label == 0 and call == 1 for label, call in zip(labels, predicted, strict=True)) + fn = sum(label == 1 and call == 0 for label, call in zip(labels, predicted, strict=True)) + precision = tp / (tp + fp) if tp + fp else 0.0 + recall = tp / (tp + fn) if tp + fn else 0.0 + f1 = 2 * precision * recall / (precision + recall) if precision + recall else 0.0 + denominator = math.sqrt((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn)) + result: dict[str, object] = { + "records": len(labels), + "positives": sum(labels), + "prevalence": sum(labels) / len(labels), + "threshold": threshold, + "confusion_matrix": {"tp": tp, "tn": tn, "fp": fp, "fn": fn}, + "precision": precision, + "recall": recall, + "f1": f1, + "mcc": (tp * tn - fp * fn) / denominator if denominator else None, + } + if len(set(labels)) == 2: + result["auroc"] = auroc(labels, oriented) + else: + result["auroc"] = None + result["average_precision"] = average_precision(labels, oriented) if sum(labels) else None + if direction == "higher" and all(0 <= score <= 1 for score in raw_scores): + result["brier_score"] = mean( + (score - label) ** 2 for label, score in zip(labels, raw_scores, strict=True) + ) + result["brier_status"] = "computed_from_probability_range_scores" + else: + result["brier_score"] = None + result["brier_status"] = "not_computed_scores_are_not_higher_is_better_probabilities" + return result + + +def _patient_values( + record_ids: set[str], submission: Submission, predictor: str +) -> dict[str, dict[str, float]]: + by_patient: dict[str, list[str]] = defaultdict(list) + for record_id in sorted(record_ids): + by_patient[submission.records[record_id].patient_id].append(record_id) + values = {} + for patient_id, patient_records in by_patient.items(): + labels = [submission.records[record_id].label for record_id in patient_records] + if not sum(labels): + continue + scores = [submission.scores[predictor][record_id] for record_id in patient_records] + if any(score is None for score in scores): + raise AssertionError("patient metrics require numeric common-support scores") + direction = submission.directions[predictor] + values[patient_id] = tie_aware_ranking_metrics( + labels, + [_oriented(float(score), direction) for score in scores], + KS, + ) + return values + + +def _patient_summary( + values: dict[str, dict[str, float]], bootstrap: int, seed: int +) -> dict[str, object]: + if not values: + return {"eligible_positive_bearing_patients": 0, "metrics": {}} + patient_ids = sorted(values) + samples: dict[str, list[float]] = {metric: [] for metric in PATIENT_METRICS} + if bootstrap: + rng = random.Random(seed) + for _ in range(bootstrap): + draw = rng.choices(patient_ids, k=len(patient_ids)) + for metric in PATIENT_METRICS: + samples[metric].append(mean(values[patient_id][metric] for patient_id in draw)) + return { + "eligible_positive_bearing_patients": len(patient_ids), + "bootstrap_unit": "patient", + "bootstrap_replicates": bootstrap, + "seed": seed, + "metrics": { + metric: { + "estimate": mean(patient[metric] for patient in values.values()), + "ci95": _interval(samples[metric]), + } + for metric in PATIENT_METRICS + }, + } + + +def _paired_differences( + patient_values: dict[str, dict[str, dict[str, float]]], bootstrap: int, seed: int +) -> list[dict[str, object]]: + results = [] + for pair_index, (left, right) in enumerate(combinations(sorted(patient_values), 2)): + patient_ids = sorted(set(patient_values[left]) & set(patient_values[right])) + if not patient_ids: + continue + rng = random.Random(seed + pair_index) + samples: dict[str, list[float]] = {metric: [] for metric in PATIENT_METRICS} + if bootstrap: + for _ in range(bootstrap): + draw = rng.choices(patient_ids, k=len(patient_ids)) + for metric in PATIENT_METRICS: + samples[metric].append( + mean( + patient_values[left][patient_id][metric] + - patient_values[right][patient_id][metric] + for patient_id in draw + ) + ) + results.append( + { + "left": left, + "right": right, + "eligible_paired_patients": len(patient_ids), + "difference": { + metric: { + "estimate": mean( + patient_values[left][patient_id][metric] + - patient_values[right][patient_id][metric] + for patient_id in patient_ids + ), + "ci95": _interval(samples[metric]), + } + for metric in PATIENT_METRICS + }, + } + ) + return results + + +def _stratified_views( + record_ids: set[str], submission: Submission, predictor: str +) -> dict[str, list[dict[str, object]]]: + result = {} + direction = submission.directions[predictor] + for field in ("study_id", "hla", "assay", "cancer_type"): + grouped: dict[str, list[str]] = defaultdict(list) + for record_id in record_ids: + value = getattr(submission.records[record_id], field) + if value: + grouped[value].append(record_id) + views = [] + for value, ids in sorted(grouped.items()): + labels = [submission.records[record_id].label for record_id in ids] + raw_scores = [submission.scores[predictor][record_id] for record_id in ids] + scores = [_oriented(float(score), direction) for score in raw_scores] + views.append( + { + "group": value, + "records": len(ids), + "patients": len( + {submission.records[record_id].patient_id for record_id in ids} + ), + "positives": sum(labels), + "auroc": auroc(labels, scores) if len(set(labels)) == 2 else None, + "average_precision": average_precision(labels, scores) if sum(labels) else None, + } + ) + if views: + result[field] = views + return result + + +def evaluate_submission( + path: Path, + *, + threshold: float = 0.5, + bootstrap: int = 1000, + seed: int = 20260820, + rank_unit: str = "pMHC", + default_direction: str = "higher", +) -> dict[str, object]: + """Evaluate a standard joined prediction file on leakage-filtered common support.""" + if not math.isfinite(threshold): + raise BenchmarkError("threshold must be finite") + if bootstrap < 0: + raise BenchmarkError("bootstrap must be non-negative") + if rank_unit not in {"pMHC", "peptide"}: + raise BenchmarkError("rank unit must be pMHC or peptide") + if rank_unit == "peptide": + raise BenchmarkError( + "peptide ranking requires an explicit cross-HLA aggregation rule; use pMHC" + ) + submission = read_submission(path, default_direction) + predictors = sorted(submission.scores) + universe = set(submission.records) + support = { + predictor: { + record_id + for record_id, score in submission.scores[predictor].items() + if score is not None + } + for predictor in predictors + } + exact = { + predictor: { + record_id + for record_id, overlap in submission.overlaps[predictor].items() + if overlap == "exact" + } + for predictor in predictors + } + audited_support = { + predictor: support[predictor] - exact[predictor] for predictor in predictors + } + raw_common = set.intersection(*(support[predictor] for predictor in predictors)) + common = set.intersection(*(audited_support[predictor] for predictor in predictors)) + if not common: + raise BenchmarkError("no leakage-filtered common support remains for evaluation") + + models = {} + patient_values = {} + for predictor in predictors: + labels = [submission.records[record_id].label for record_id in sorted(common)] + scores = [float(submission.scores[predictor][record_id]) for record_id in sorted(common)] + values = _patient_values(common, submission, predictor) + patient_values[predictor] = values + status_counts = { + status: sum(value == status for value in submission.statuses[predictor].values()) + for status in sorted(VALID_STATUSES) + } + overlap_counts = { + overlap: sum(value == overlap for value in submission.overlaps[predictor].values()) + for overlap in sorted(VALID_OVERLAP) + } + models[predictor] = { + "score_direction": submission.directions[predictor], + "coverage": len(support[predictor]) / len(universe), + "submitted_records": len(submission.scores[predictor]), + "omitted_records": len(universe - set(submission.scores[predictor])), + "supported_records": len(support[predictor]), + "status_counts": status_counts, + "training_overlap": overlap_counts, + "exact_overlap_excluded": len(exact[predictor]), + "pooled_common_support": _classification_metrics( + labels, scores, submission.directions[predictor], threshold + ), + "patient_common_support": _patient_summary(values, bootstrap, seed), + "stratified_descriptive": _stratified_views(common, submission, predictor), + } + + random_values = {} + by_patient: dict[str, list[str]] = defaultdict(list) + for record_id in common: + by_patient[submission.records[record_id].patient_id].append(record_id) + for patient_id, record_ids in by_patient.items(): + labels = [submission.records[record_id].label for record_id in record_ids] + if sum(labels): + random_values[patient_id] = tie_aware_ranking_metrics( + labels, [0.0] * len(labels), KS + ) + + return { + "schema_version": 1, + "protocol": { + "score_direction": "declared_per_predictor; default higher", + "classification_threshold": threshold, + "rank_unit": rank_unit, + "top_k": list(KS), + "tie_rule": "analytic expectation over tied-score permutations", + "primary_comparison": "exact-overlap-filtered common support", + "bootstrap": { + "unit": "patient", + "replicates": bootstrap, + "seed": seed, + }, + "interpretation": ( + "descriptive discrimination and prioritization; not clinical efficacy or " + "held-out generalization" + ), + }, + "input": { + "path": str(path), + "sha256": submission.sha256, + "rows": submission.rows, + "records": len(universe), + "predictors": predictors, + "columns": list(submission.columns), + }, + "environment": { + "python": platform.python_version(), + "platform": platform.platform(), + }, + "support": { + "raw_common_records": len(raw_common), + "leakage_filtered_common_records": len(common), + "leakage_filtered_common_coverage": len(common) / len(universe), + }, + "models": models, + "support_matched_random_ranking": _patient_summary(random_values, bootstrap, seed), + "paired_patient_differences": _paired_differences(patient_values, bootstrap, seed), + "limitations": [ + "Unknown training overlap remains unknown and is not evidence of independence.", + "Stratified results are descriptive; this command does not fit held-out folds.", + "Brier score is computed only for higher-is-better scores entirely in [0, 1].", + "Clinical efficacy cannot be inferred from predictor benchmark performance.", + ], + } + + +def markdown_report(result: dict[str, object]) -> str: + """Render a concise, self-contained human-readable benchmark report.""" + support = result["support"] + protocol = result["protocol"] + lines = [ + "# NeoRepro standard benchmark report", + "", + f"Input SHA-256: `{result['input']['sha256']}`", + "", + ( + f"Primary comparison uses **{support['leakage_filtered_common_records']}** " + f"records ({support['leakage_filtered_common_coverage']:.1%}) on " + "exact-overlap-filtered common support." + ), + "", + "| Predictor | Coverage | AUROC | AUPRC | Recall@20 | NDCG@5 |", + "|---|---:|---:|---:|---:|---:|", + ] + for predictor, model in result["models"].items(): + pooled = model["pooled_common_support"] + patient = model["patient_common_support"]["metrics"] + + def value(metric: object) -> str: + return "NA" if metric is None else f"{float(metric):.3f}" + + lines.append( + f"| {predictor} | {model['coverage']:.1%} | {value(pooled['auroc'])} | " + f"{value(pooled['average_precision'])} | " + f"{value(patient.get('recall@20', {}).get('estimate'))} | " + f"{value(patient.get('ndcg@5', {}).get('estimate'))} |" + ) + lines += [ + "", + "## Evaluation contract", + "", + f"- Classification threshold: `{protocol['classification_threshold']}`.", + f"- Ranking unit: `{protocol['rank_unit']}`; Top-K: `{protocol['top_k']}`.", + f"- Tie handling: {protocol['tie_rule']}.", + ( + f"- Patient bootstrap: {protocol['bootstrap']['replicates']} replicates, " + f"seed `{protocol['bootstrap']['seed']}`." + ), + "- Missing or failed predictions are reported and never imputed.", + "- Exact declared training overlaps are excluded; unknown overlap remains unknown.", + "", + "## Interpretation limits", + "", + ] + lines.extend(f"- {limitation}" for limitation in result["limitations"]) + return "\n".join(lines) + "\n" + + +def run_benchmark( + path: Path, + output_dir: Path, + *, + threshold: float = 0.5, + bootstrap: int = 1000, + seed: int = 20260820, + rank_unit: str = "pMHC", + default_direction: str = "higher", +) -> tuple[dict[str, object], Path, Path]: + """Evaluate one submission and atomically publish JSON and Markdown outputs.""" + result = evaluate_submission( + path, + threshold=threshold, + bootstrap=bootstrap, + seed=seed, + rank_unit=rank_unit, + default_direction=default_direction, + ) + output_dir.mkdir(parents=True, exist_ok=True) + json_path = output_dir / "evaluation.json" + report_path = output_dir / "report.md" + json_temp = output_dir / ".evaluation.json.tmp" + report_temp = output_dir / ".report.md.tmp" + json_temp.write_text( + json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", + encoding="utf-8", + ) + report_temp.write_text(markdown_report(result), encoding="utf-8") + json_temp.replace(json_path) + report_temp.replace(report_path) + return result, json_path, report_path diff --git a/src/neorepro/cli.py b/src/neorepro/cli.py index a0cfdae..6db4f9c 100644 --- a/src/neorepro/cli.py +++ b/src/neorepro/cli.py @@ -4,11 +4,26 @@ import argparse import csv +import json from pathlib import Path +from neorepro.audit import AuditError, audit_predictions +from neorepro.benchmark import BenchmarkError, run_benchmark +from neorepro.contract import ( + ContractError, + evaluate, + markdown_report, + validate_artifact, + validate_card, +) + def project_root() -> Path: - return Path.cwd() + cwd = Path.cwd() + if (cwd / "RESEARCH_SPEC.md").exists(): + return cwd + # Keep the legacy audit usable when invoked from another working directory. + return Path(__file__).resolve().parents[2] def list_predictors(root: Path) -> int: @@ -42,15 +57,135 @@ def main(argv: list[str] | None = None) -> int: subparsers = parser.add_subparsers(dest="command", required=True) subparsers.add_parser("list-predictors") subparsers.add_parser("status") + audit_parser = subparsers.add_parser( + "audit", + help="audit external prediction scores against frozen benchmark evidence", + description=( + "Audit a CSV containing exactly patient_id, peptide, hla, score, model. " + "Scores must be numeric or blank for unsupported rows; higher is better." + ), + ) + audit_parser.add_argument("predictions", type=Path, help="five-column prediction CSV") + benchmark_parser = subparsers.add_parser( + "benchmark", + help="run the portable standard benchmark from one joined prediction CSV", + description=( + "Evaluate one or more predictors from a CSV containing record_id, patient_id, " + "study_id, label, score and predictor. Produces evaluation.json and report.md." + ), + ) + benchmark_parser.add_argument("predictions", type=Path) + benchmark_parser.add_argument( + "--output-dir", type=Path, default=Path("neorepro-results") + ) + benchmark_parser.add_argument("--threshold", type=float, default=0.5) + benchmark_parser.add_argument("--bootstrap", type=int, default=1000) + benchmark_parser.add_argument("--seed", type=int, default=20260820) + benchmark_parser.add_argument( + "--score-direction", choices=["higher", "lower"], default="higher" + ) + benchmark_parser.add_argument("--rank-unit", choices=["pMHC", "peptide"], default="pMHC") + + def add_path_command(name, help_text): + p = subparsers.add_parser(name, help=help_text) + p.add_argument("path", type=Path) + return p + + dataset_parser = subparsers.add_parser("dataset", help="validate a Dataset Card JSON") + dataset_parser.add_argument("action", choices=["validate"]) + dataset_parser.add_argument("path", type=Path) + predictor_parser = subparsers.add_parser("predictor", help="validate a Predictor Card JSON") + predictor_parser.add_argument("action", choices=["validate"]) + predictor_parser.add_argument("path", type=Path) + artifact_parser = add_path_command("artifact", "validate a prediction artifact CSV") + artifact_parser.add_argument("--benchmark", type=Path) + overlap_parser = add_path_command( + "overlap-audit", "run the frozen legacy overlap/common-support audit" + ) + overlap_parser.add_argument("--root", type=Path, default=None) + evaluate_parser = subparsers.add_parser("evaluate", help="evaluate artifacts on common support") + evaluate_parser.add_argument("benchmark", type=Path) + evaluate_parser.add_argument("artifacts", nargs="+", type=Path) + evaluate_parser.add_argument("--overlap-audit", type=Path) + evaluate_parser.add_argument( + "--output", type=Path, default=Path("results/contract_evaluation.json") + ) + evaluate_parser.add_argument( + "--report", type=Path, default=Path("reports/contract_evaluation.md") + ) + report_parser = subparsers.add_parser("report", help="render an evaluation JSON as Markdown") + report_parser.add_argument("evaluation", type=Path) + report_parser.add_argument( + "--output", type=Path, default=Path("reports/contract_evaluation.md") + ) args = parser.parse_args(argv) root = project_root() if args.command == "list-predictors": return list_predictors(root) if args.command == "status": return status(root) + if args.command == "audit": + try: + report = audit_predictions(args.predictions, root) + except AuditError as error: + parser.error(str(error)) + print(json.dumps(report, indent=2, sort_keys=True, allow_nan=False)) + return 0 + if args.command == "benchmark": + try: + result, json_path, report_path = run_benchmark( + args.predictions, + args.output_dir, + threshold=args.threshold, + bootstrap=args.bootstrap, + seed=args.seed, + rank_unit=args.rank_unit, + default_direction=args.score_direction, + ) + except (BenchmarkError, OSError) as error: + parser.error(str(error)) + print( + json.dumps( + { + "status": "ok", + "predictors": result["input"]["predictors"], + "common_support": result["support"][ + "leakage_filtered_common_records" + ], + "evaluation": str(json_path), + "report": str(report_path), + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + try: + if args.command == "dataset": + print(json.dumps(validate_card(args.path, "dataset"), indent=2)) + return 0 + if args.command == "predictor": + print(json.dumps(validate_card(args.path, "predictor"), indent=2)) + return 0 + if args.command == "artifact": + print(json.dumps(validate_artifact(args.path, args.benchmark), indent=2)) + return 0 + if args.command == "overlap-audit": + report = audit_predictions(args.path, args.root or root) + print(json.dumps(report, indent=2, sort_keys=True)) + return 0 + if args.command == "evaluate": + result = evaluate(args.benchmark, args.artifacts, args.output, args.overlap_audit) + markdown_report(result, args.report) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + if args.command == "report": + markdown_report(json.loads(args.evaluation.read_text()), args.output) + return 0 + except (ContractError, OSError, KeyError, ValueError) as error: + parser.error(str(error)) raise AssertionError(args.command) if __name__ == "__main__": raise SystemExit(main()) - diff --git a/src/neorepro/contract.py b/src/neorepro/contract.py new file mode 100644 index 0000000..19d5fb9 --- /dev/null +++ b/src/neorepro/contract.py @@ -0,0 +1,211 @@ +"""Public extension contract: cards, artifacts, gates, evaluation and reports.""" + +from __future__ import annotations + +import csv +import json +import math +from collections import defaultdict +from pathlib import Path + +from neorepro.metrics import auroc, tie_aware_ranking_metrics + +DATASET_REQUIRED = { + "dataset_id", + "version", + "records_path", + "label_column", + "patient_id_column", + "score_tasks", +} +PREDICTOR_REQUIRED = {"predictor_id", "version", "task", "score_direction", "adapter", "license"} +ARTIFACT_REQUIRED = { + "record_id", + "predictor", + "predictor_version", + "task", + "score", + "score_direction", + "status", +} +STATUSES = {"predicted", "unsupported", "failed", "invalid"} + + +class ContractError(ValueError): + pass + + +def _json(path): + try: + return json.loads(Path(path).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + raise ContractError(f"invalid JSON: {path}: {e}") from e + + +def validate_card(path: Path, kind: str) -> dict: + if kind not in {"dataset", "predictor"}: + raise ContractError(f"unknown card kind: {kind}") + obj = _json(path) + required = DATASET_REQUIRED if kind == "dataset" else PREDICTOR_REQUIRED + missing = sorted(required - obj.keys()) + if missing: + raise ContractError(f"{kind} card missing fields: {', '.join(missing)}") + if not isinstance(obj["version"], str) or not obj["version"]: + raise ContractError("card version must be non-empty") + if kind == "predictor" and obj["score_direction"] not in {"higher", "lower"}: + raise ContractError("score_direction must be higher or lower") + return { + "valid": True, + "kind": kind, + "path": str(path), + "id": obj.get("dataset_id", obj.get("predictor_id")), + "version": obj["version"], + } + + +def read_csv(path): + with Path(path).open(newline="", encoding="utf-8-sig") as f: + r = csv.DictReader(f) + if not r.fieldnames: + raise ContractError(f"CSV has no header: {path}") + return list(r), list(r.fieldnames) + + +def validate_artifact(path: Path, benchmark: Path | None = None) -> dict: + rows, header = read_csv(path) + missing = sorted(ARTIFACT_REQUIRED - set(header)) + if missing: + raise ContractError(f"prediction artifact missing columns: {', '.join(missing)}") + if not rows: + raise ContractError("prediction artifact has no rows") + ids = set() + problems = [] + for n, row in enumerate(rows, 2): + if not row["record_id"] or row["record_id"] in ids: + problems.append(f"line {n}: duplicate/blank record_id") + ids.add(row["record_id"]) + if row["status"] not in STATUSES: + problems.append(f"line {n}: unknown status {row['status']}") + if row["score_direction"] not in {"higher", "lower"}: + problems.append(f"line {n}: invalid score_direction") + if row["status"] == "predicted": + try: + if not math.isfinite(float(row["score"])): + raise ValueError + except ValueError: + problems.append(f"line {n}: predicted score must be finite") + if benchmark: + brows, _ = read_csv(benchmark) + expected = {r["record_id"] for r in brows} + if ids != expected: + problems.append( + f"record support mismatch: submitted={len(ids)} expected={len(expected)}" + ) + if problems: + raise ContractError("; ".join(problems[:8])) + return { + "valid": True, + "rows": len(rows), + "predictors": sorted({r["predictor"] for r in rows}), + "tasks": sorted({r["task"] for r in rows}), + "predicted": sum(r["status"] == "predicted" for r in rows), + "missing": sum(r["status"] != "predicted" for r in rows), + } + + +def evaluate( + benchmark: Path, artifacts: list[Path], output: Path, overlap_audit: Path | None = None +) -> dict: + b, _ = read_csv(benchmark) + labels = {r["record_id"]: int(r["label"] if "label" in r else r["immunogenicity"]) for r in b} + patients = {r["record_id"]: r.get("patient_id", "unknown") for r in b} + result = { + "schema_version": 1, + "benchmark": str(benchmark), + "models": {}, + "gates": {"common_support": True, "missingness": True, "leakage": "not_checked"}, + } + if overlap_audit: + audit_rows, audit_header = read_csv(overlap_audit) + if "record_id" not in audit_header: + raise ContractError("overlap audit must contain record_id") + audit_ids = {r["record_id"] for r in audit_rows} + if not audit_ids <= set(labels): + raise ContractError("overlap audit contains record_id absent from benchmark") + result["gates"]["leakage"] = { + "status": "checked", + "audit": str(overlap_audit), + "rows": len(audit_rows), + } + parsed = [] + for p in artifacts: + validate_artifact(p, benchmark) + rows, _ = read_csv(p) + parsed.append(rows) + supports = [{r["record_id"] for r in rows if r["status"] == "predicted"} for rows in parsed] + common = set.intersection(*supports) if supports else set() + result["common_support"] = { + "records": len(common), + "coverage": len(common) / len(labels) if labels else 0, + } + for rows, support in zip(parsed, supports): + name = rows[0]["predictor"] + scores = {r["record_id"]: float(r["score"]) for r in rows if r["status"] == "predicted"} + use = common + direction = rows[0]["score_direction"] + vals = [scores[k] if direction == "higher" else -scores[k] for k in use] + labs = [labels[k] for k in use] + model = { + "rows": len(rows), + "predicted": len(support), + "missing": len(set(labels) - support), + "coverage": len(support) / len(labels), + "task": rows[0]["task"], + "score_direction": direction, + } + if len(set(labs)) == 2: + model["auroc"] = auroc(labs, vals) + by = defaultdict(list) + for k in use: + by[patients[k]].append(k) + pm = [] + for ks in by.values(): + if any(labels[k] for k in ks): + pm.append( + tie_aware_ranking_metrics( + [labels[k] for k in ks], + [scores[k] if direction == "higher" else -scores[k] for k in ks], + [5], + ) + ) + if pm: + model["patient_ndcg@5"] = sum(x["ndcg@5"] for x in pm) / len(pm) + result["models"][name] = model + output.parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return result + + +def markdown_report(result: dict, output: Path): + lines = [ + "# NeoRepro evaluation report", + "", + f"Common support: **{result['common_support']['records']}** records ({result['common_support']['coverage']:.1%}).", + "", + "| Predictor | Task | Coverage | AUROC | Patient NDCG@5 |", + "|---|---|---:|---:|---:|", + ] + for name, m in result["models"].items(): + lines.append( + f"| {name} | {m['task']} | {m['coverage']:.1%} | {m.get('auroc', 'unknown')} | {m.get('patient_ndcg@5', 'unknown')} |" + ) + lines += [ + "", + "## Gates", + "", + "- Common support: passed for reported comparisons.", + "- Missingness: reported per predictor; no failed row was imputed.", + "- Leakage: run `neorepro overlap-audit`; unknown training overlap remains unknown.", + ] + output.parent.mkdir(parents=True, exist_ok=True) + Path(output).write_text("\n".join(lines) + "\n", encoding="utf-8") diff --git a/src/neorepro/metrics.py b/src/neorepro/metrics.py index c1338cb..e24efdd 100644 --- a/src/neorepro/metrics.py +++ b/src/neorepro/metrics.py @@ -4,6 +4,7 @@ import math from collections.abc import Iterable, Sequence +from math import comb def _validate(labels: Sequence[int], scores: Sequence[float]) -> None: @@ -82,3 +83,70 @@ def ranking_metrics(labels_in_rank_order: Sequence[int], ks: Iterable[int]) -> d idcg = sum(1 / math.log2(rank + 1) for rank in range(1, ideal_hits + 1)) result[f"ndcg@{k}"] = dcg / idcg return result + + +def tie_aware_ranking_metrics( + labels: Sequence[int], scores: Sequence[float], ks: Iterable[int] +) -> dict[str, float]: + """Compute expected patient-ranking metrics over every tied-score ordering.""" + _validate(labels, scores) + positives = sum(labels) + if not positives: + raise ValueError("ranking metrics require a positive-bearing patient") + ordered = sorted(range(len(scores)), key=scores.__getitem__, reverse=True) + score_groups: list[tuple[int, int]] = [] + start = 0 + while start < len(ordered): + end = start + 1 + while end < len(ordered) and scores[ordered[end]] == scores[ordered[start]]: + end += 1 + score_groups.append((end - start, sum(labels[index] for index in ordered[start:end]))) + start = end + + offset = 0 + expected_mrr = 0.0 + for size, group_positives in score_groups: + if group_positives: + denominator = comb(size, group_positives) + expected_mrr = sum( + (comb(size - first, group_positives - 1) / denominator) / (offset + first) + for first in range(1, size - group_positives + 2) + ) + break + offset += size + + result = {"mrr": expected_mrr} + for k in ks: + if k <= 0: + raise ValueError("K must be positive") + limit = min(k, len(ordered)) + remaining = limit + offset = 0 + expected_hits = 0.0 + expected_dcg = 0.0 + zero_hit_probability = 1.0 + for size, group_positives in score_groups: + if not remaining: + break + selected = min(remaining, size) + expected_hits += selected * group_positives / size + expected_dcg += (group_positives / size) * sum( + 1 / math.log2(rank + 1) + for rank in range(offset + 1, offset + selected + 1) + ) + if selected == size: + if group_positives: + zero_hit_probability = 0.0 + elif zero_hit_probability and group_positives: + zero_hit_probability *= comb(size - group_positives, selected) / comb( + size, selected + ) + remaining -= selected + offset += selected + result[f"recall@{k}"] = expected_hits / positives + result[f"precision@{k}"] = expected_hits / limit + result[f"hitrate@{k}"] = 1 - zero_hit_probability + ideal_hits = min(positives, limit) + idcg = sum(1 / math.log2(rank + 1) for rank in range(1, ideal_hits + 1)) + result[f"ndcg@{k}"] = expected_dcg / idcg + return result diff --git a/tests/test_audit_cli.py b/tests/test_audit_cli.py new file mode 100644 index 0000000..c9c8431 --- /dev/null +++ b/tests/test_audit_cli.py @@ -0,0 +1,111 @@ +import csv +import json +from pathlib import Path + +import pytest + +from neorepro.audit import AuditError, audit_predictions +from neorepro.cli import main + +ROOT = Path(__file__).resolve().parents[1] +TESLA = ROOT / "data/processed/benchmark.csv" +ZHAO = ROOT / "data/processed/zhao_vaccine_benchmark_full.csv" +FIELDS = ["patient_id", "peptide", "hla", "score", "model"] + + +def read_rows(path: Path) -> list[dict[str, str]]: + with path.open(newline="", encoding="utf-8-sig") as handle: + return list(csv.DictReader(handle)) + + +def write_predictions(path: Path, rows: list[dict[str, str]]) -> None: + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + + +def prediction(source: dict[str, str], score: str, model: str) -> dict[str, str]: + return { + "patient_id": source["patient_id"], + "peptide": source["peptide"], + "hla": source["hla"], + "score": score, + "model": model, + } + + +def test_audit_reports_leakage_support_and_patient_ci(tmp_path: Path) -> None: + source = read_rows(TESLA)[:3] + path = tmp_path / "predictions.csv" + write_predictions( + path, + [ + prediction(source[0], "0.1", "External A"), + prediction(source[1], "0.2", "External A"), + prediction(source[2], "0.9", "External A"), + prediction(source[0], "0.1", "External B"), + prediction(source[1], "", "External B"), + prediction(source[2], "0.8", "External B"), + ], + ) + + report = audit_predictions(path, ROOT) + + assert report["benchmark"]["datasets"] == ["TESLA"] + assert report["benchmark"]["expected_unique_patient_peptide_hla_records"] == 520 + assert report["leakage"]["External A"]["risk"] == "unknown_training_reference" + assert report["support"]["by_model"]["External A"]["supported_records"] == 3 + assert report["support"]["by_model"]["External B"]["blank_score_records"] == 1 + assert report["support"]["raw_common_support"]["records"] == 2 + metric = report["patient_metrics"]["models"]["External A"]["metrics"]["ndcg@5"] + assert metric["estimate"] == pytest.approx(1.0) + assert metric["ci95"] == {"low": pytest.approx(1.0), "high": pytest.approx(1.0)} + + +def test_known_exact_overlap_is_excluded_from_metrics(tmp_path: Path) -> None: + source = read_rows(TESLA)[:3] + path = tmp_path / "prime.csv" + write_predictions( + path, + [prediction(row, str(index), "PRIME-2.0") for index, row in enumerate(source)], + ) + + report = audit_predictions(path, ROOT) + + leakage = report["leakage"]["PRIME-2.0"] + assert leakage["risk"] == "high_exact_overlap" + assert leakage["exact_overlap_records"] == 3 + assert leakage["excluded_from_metrics_as_exact_overlap"] == 3 + assert report["support"]["leakage_filtered_common_support"]["records"] == 0 + assert report["patient_metrics"]["models"]["PRIME-2.0"]["metrics"] == {} + + +def test_five_column_key_collapses_concordant_source_duplicates(tmp_path: Path) -> None: + source = read_rows(ZHAO)[0] + path = tmp_path / "zhao.csv" + write_predictions(path, [prediction(source, "0.5", "External")]) + + report = audit_predictions(path, ROOT) + + assert report["benchmark"]["expected_unique_patient_peptide_hla_records"] == 2315 + assert report["benchmark"]["collapsed_duplicate_source_rows"] == 2 + + +def test_audit_rejects_non_contract_header(tmp_path: Path) -> None: + path = tmp_path / "bad.csv" + path.write_text("patient_id,peptide,hla,score,model,label\n", encoding="utf-8") + + with pytest.raises(AuditError, match="header must contain exactly"): + audit_predictions(path, ROOT) + + +def test_cli_prints_json_report(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + source = read_rows(TESLA)[0] + path = tmp_path / "predictions.csv" + write_predictions(path, [prediction(source, "0.5", "External")]) + + assert main(["audit", str(path)]) == 0 + output = json.loads(capsys.readouterr().out) + assert output["input"]["columns"] == FIELDS + assert output["input"]["score_direction"] == "higher_is_better" diff --git a/tests/test_contract.py b/tests/test_contract.py new file mode 100644 index 0000000..9fc9516 --- /dev/null +++ b/tests/test_contract.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import pytest + +from neorepro.contract import ContractError, evaluate, validate_artifact, validate_card + +ROOT = Path(__file__).resolve().parents[1] + + +def test_cards_and_artifact(): + assert validate_card(ROOT / "contracts/dataset-card.example.json", "dataset")["valid"] + assert validate_card(ROOT / "contracts/predictor-card.example.json", "predictor")["valid"] + assert ( + validate_artifact( + ROOT / "contracts/synthetic/predictions.csv", ROOT / "contracts/synthetic/benchmark.csv" + )["missing"] + == 1 + ) + + +def test_evaluate_synthetic(tmp_path): + out = tmp_path / "eval.json" + result = evaluate( + ROOT / "contracts/synthetic/benchmark.csv", + [ROOT / "contracts/synthetic/predictions.csv"], + out, + ) + assert result["common_support"]["records"] == 3 + assert result["models"]["synthetic"]["auroc"] == 1.0 + + +def test_lower_direction_and_overlap_gate(tmp_path): + lower = """record_id,predictor,predictor_version,task,score,score_direction,status +s1,synthetic,1.0,immunogenicity,0.1,lower,predicted +s2,synthetic,1.0,immunogenicity,0.9,lower,predicted +s3,synthetic,1.0,immunogenicity,0.2,lower,predicted +s4,synthetic,1.0,immunogenicity,,lower,unsupported +""" + path = tmp_path / "lower.csv" + path.write_text(lower) + audit = tmp_path / "overlap.csv" + audit.write_text("record_id,exact_overlap\ns1,0\ns2,0\ns3,0\ns4,0\n") + result = evaluate( + ROOT / "contracts/synthetic/benchmark.csv", [path], tmp_path / "out.json", audit + ) + assert result["models"]["synthetic"]["score_direction"] == "lower" + assert result["models"]["synthetic"]["auroc"] == 1.0 + assert result["gates"]["leakage"]["status"] == "checked" + + +def test_artifact_rejects_duplicate_or_missing_support(tmp_path): + rows = (ROOT / "contracts/synthetic/predictions.csv").read_text().splitlines() + path = tmp_path / "bad.csv" + path.write_text("\n".join(rows[:-1] + [rows[1]]) + "\n") + with pytest.raises(ContractError, match="duplicate|support mismatch"): + validate_artifact(path, ROOT / "contracts/synthetic/benchmark.csv") diff --git a/tests/test_evaluator_ties.py b/tests/test_evaluator_ties.py index 30483b5..35e3700 100644 --- a/tests/test_evaluator_ties.py +++ b/tests/test_evaluator_ties.py @@ -1,5 +1,7 @@ import importlib.util +import json import math +import random from pathlib import Path import pytest @@ -30,3 +32,19 @@ def test_patient_top_k_uses_analytic_expectation_for_ties() -> None: assert values["mrr"] == pytest.approx(sum(1 / rank for rank in range(1, 11)) / 10) expected_dcg = 0.1 * sum(1 / math.log2(rank + 1) for rank in range(1, 6)) assert values["ndcg@5"] == pytest.approx(expected_dcg) + + +def test_bootstrap_checkpoint_helpers_are_atomic_and_input_bound(tmp_path: Path) -> None: + source = tmp_path / "input.csv" + source.write_text("a,b\n1,2\n") + first = MODULE.input_fingerprint([source], 2000, 7) + assert first == MODULE.input_fingerprint([source], 2000, 7) + assert first != MODULE.input_fingerprint([source], 2000, 8) + source.write_text("a,b\n1,3\n") + assert first != MODULE.input_fingerprint([source], 2000, 7) + + checkpoint = tmp_path / ".bootstrap_checkpoint.json" + state = {"completed": 50, "rng_state": repr(random.Random(7).getstate())} + MODULE.atomic_json(checkpoint, state) + assert json.loads(checkpoint.read_text()) == state + assert not checkpoint.with_suffix(".json.tmp").exists() diff --git a/tests/test_metrics.py b/tests/test_metrics.py index 2ed37ed..bfb01eb 100644 --- a/tests/test_metrics.py +++ b/tests/test_metrics.py @@ -2,7 +2,12 @@ import pytest -from neorepro.metrics import auroc, average_precision, ranking_metrics +from neorepro.metrics import ( + auroc, + average_precision, + ranking_metrics, + tie_aware_ranking_metrics, +) def test_perfect_pooled_ranking() -> None: @@ -33,3 +38,10 @@ def test_patient_top_k_uses_available_candidates() -> None: def test_ranking_requires_positive_patient() -> None: with pytest.raises(ValueError, match="positive-bearing"): ranking_metrics([0, 0], [5]) + + +def test_tie_aware_ranking_uses_expected_ordering() -> None: + result = tie_aware_ranking_metrics([1, 0], [0.5, 0.5], [1]) + assert result["recall@1"] == pytest.approx(0.5) + assert result["hitrate@1"] == pytest.approx(0.5) + assert result["mrr"] == pytest.approx(0.75) diff --git a/tests/test_rcc_extension.py b/tests/test_rcc_extension.py new file mode 100644 index 0000000..84a665d --- /dev/null +++ b/tests/test_rcc_extension.py @@ -0,0 +1,41 @@ +import csv +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def load_csv(path: str) -> list[dict[str, str]]: + with (ROOT / path).open(newline="", encoding="utf-8") as handle: + return list(csv.DictReader(handle)) + + +def test_rcc_benchmark_and_overlap_contract() -> None: + rows = load_csv("data/processed/rcc_vaccine_benchmark.csv") + assert len(rows) == 129 + assert len({row["patient_id"] for row in rows}) == 9 + assert sum(int(row["immunogenicity"]) for row in rows) == 75 + summary = json.loads((ROOT / "research/training_overlap_summary_rcc.json").read_text()) + assert summary["benchmark_union_known_exact_overlap"] == 0 + assert summary["overlap_dimensions"]["deephlapan_training_identity"].startswith("unknown_") + + +def test_rcc_predictions_are_complete_artifacts() -> None: + paths = sorted((ROOT / "results/raw_predictions/rcc").glob("*.csv")) + assert len(paths) == 4 + for path in paths: + rows = load_csv(str(path.relative_to(ROOT))) + assert len(rows) == 129 + assert len({row["record_id"] for row in rows}) == 129 + assert {row["status"] for row in rows} <= { + "predicted", + "unsupported_hla", + "unsupported_length", + } + + +def test_generated_manuscript_contains_rcc_table() -> None: + manuscript = (ROOT / "paper/manuscript_resource.md").read_text(encoding="utf-8") + assert "### Endpoint-distinct RCC vaccine cohort" in manuscript + assert "**Table 4. RCC personalized-vaccine cohort.**" in manuscript + assert "[@braun2025rcc]" in manuscript diff --git a/tests/test_stability_analysis.py b/tests/test_stability_analysis.py new file mode 100644 index 0000000..b4e65e8 --- /dev/null +++ b/tests/test_stability_analysis.py @@ -0,0 +1,48 @@ +import csv +import json +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "results/analysis/stability" + + +class StabilityAnalysisTest(unittest.TestCase): + def test_stability_outputs_have_required_artifacts(self): + required = [ + "dataset_predictor_metric_matrix.csv", + "rank_stability.csv", + "model_selection_first_probability.csv", + "sensitivity_summary.csv", + "leave_one_domain_out.csv", + "endpoint_domain_metadata.csv", + "analysis_metadata.json", + ] + self.assertTrue(all((OUT / name).exists() for name in required)) + meta = json.loads((OUT / "analysis_metadata.json").read_text()) + self.assertEqual(meta["analysis_type"], "exploratory_descriptive_heterogeneity") + self.assertEqual(meta["bootstrap"], 2000) + self.assertEqual( + set(meta["datasets"]), + { + "improve_benchmark", + "zhao_vaccine_benchmark", + "rcc_vaccine_benchmark", + }, + ) + + def test_task_specific_first_probabilities_sum_to_one(self): + rows = list(csv.DictReader((OUT / "model_selection_first_probability.csv").open())) + for key in {(r["dataset"], r["task"], r["metric"]) for r in rows}: + group = [ + float(r["probability_first"]) + for r in rows + if (r["dataset"], r["task"], r["metric"]) == key + ] + self.assertAlmostEqual(sum(group), 1.0) + + def test_lodo_is_explicitly_descriptive(self): + rows = list(csv.DictReader((OUT / "leave_one_domain_out.csv").open())) + self.assertTrue( + rows and {r["analysis_type"] for r in rows} == {"descriptive_leave_one_domain_out"} + ) diff --git a/tests/test_standard_benchmark.py b/tests/test_standard_benchmark.py new file mode 100644 index 0000000..158f79b --- /dev/null +++ b/tests/test_standard_benchmark.py @@ -0,0 +1,74 @@ +import json +from pathlib import Path + +import pytest + +from neorepro.benchmark import BenchmarkError, evaluate_submission, run_benchmark +from neorepro.cli import main + +ROOT = Path(__file__).resolve().parents[1] +EXAMPLE = ROOT / "contracts/synthetic/standard_predictions.csv" + + +def test_standard_benchmark_reports_full_contract(tmp_path: Path) -> None: + result, json_path, report_path = run_benchmark( + EXAMPLE, tmp_path / "result", bootstrap=50, seed=7 + ) + + assert result["support"]["raw_common_records"] == 3 + assert result["support"]["leakage_filtered_common_records"] == 3 + assert result["models"]["Example-A"]["pooled_common_support"]["auroc"] == 1.0 + assert result["models"]["Example-A"]["patient_common_support"]["metrics"][ + "recall@20" + ]["estimate"] == 1.0 + assert result["models"]["Example-B"]["status_counts"]["unsupported"] == 1 + assert result["support_matched_random_ranking"]["metrics"]["ndcg@5"] + assert len(result["paired_patient_differences"]) == 1 + assert json.loads(json_path.read_text())["schema_version"] == 1 + assert "NeoRepro standard benchmark report" in report_path.read_text() + + +def test_exact_overlap_is_removed_from_common_support(tmp_path: Path) -> None: + text = EXAMPLE.read_text().replace( + "s1,P1,SYNTHETIC,1,0.9,Example-A,higher,predicted,none", + "s1,P1,SYNTHETIC,1,0.9,Example-A,higher,predicted,exact", + ) + path = tmp_path / "overlap.csv" + path.write_text(text) + + result = evaluate_submission(path, bootstrap=0) + + assert result["models"]["Example-A"]["exact_overlap_excluded"] == 1 + assert result["support"]["leakage_filtered_common_records"] == 2 + + +def test_standard_benchmark_rejects_conflicting_truth(tmp_path: Path) -> None: + path = tmp_path / "bad.csv" + path.write_text( + "record_id,patient_id,study_id,label,score,predictor\n" + "r1,p1,s1,1,0.9,A\n" + "r1,p1,s1,0,0.8,B\n" + ) + + with pytest.raises(BenchmarkError, match="conflicting truth"): + evaluate_submission(path) + + +def test_standard_benchmark_cli_writes_outputs( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + output_dir = tmp_path / "cli" + assert main( + [ + "benchmark", + str(EXAMPLE), + "--output-dir", + str(output_dir), + "--bootstrap", + "10", + ] + ) == 0 + summary = json.loads(capsys.readouterr().out) + assert summary["status"] == "ok" + assert (output_dir / "evaluation.json").exists() + assert (output_dir / "report.md").exists()