From badfae08e4a3a8970f294c2d9c4d8a867830e745 Mon Sep 17 00:00:00 2001 From: lipluscodex <268560960+lipluscodex@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:31:25 +0900 Subject: [PATCH 1/2] eval: freeze longitudinal feedback protocol (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 固定horizonと複数clusterのlongitudinal feedback protocolを追加する。\ncontrol/treatment、ceiling-aware gate、識別子限定のcontamination audit、one-shot outputを結果観測前に固定する。 --- README.md | 2 + docs/longitudinal-feedback-experiment.md | 44 +++ docs/requirements.md | 2 + .../longitudinal_feedback_experiment.py | 373 ++++++++++++++++++ ...s_longitudinal_feedback.contamination.json | 371 +++++++++++++++++ ...nal_feedback_development_ceiling.gold.json | 63 +++ ...itudinal_feedback_development_ceiling.json | 51 +++ ...edback_development_ceiling.provenance.json | 18 + ..._feedback_development_headroom-a.gold.json | 63 +++ ...dinal_feedback_development_headroom-a.json | 51 +++ ...ack_development_headroom-a.provenance.json | 18 + ..._feedback_development_headroom-b.gold.json | 63 +++ ...dinal_feedback_development_headroom-b.json | 51 +++ ...ack_development_headroom-b.provenance.json | 18 + ...itudinal_feedback_experiment.manifest.json | 139 +++++++ ...tudinal_feedback_holdout_ceiling.gold.json | 63 +++ ...longitudinal_feedback_holdout_ceiling.json | 51 +++ ...l_feedback_holdout_ceiling.provenance.json | 18 + ...inal_feedback_holdout_headroom-a.gold.json | 63 +++ ...gitudinal_feedback_holdout_headroom-a.json | 51 +++ ...eedback_holdout_headroom-a.provenance.json | 18 + ...inal_feedback_holdout_headroom-b.gold.json | 63 +++ ...gitudinal_feedback_holdout_headroom-b.json | 51 +++ ...eedback_holdout_headroom-b.provenance.json | 18 + .../test_longitudinal_feedback_experiment.py | 70 ++++ tools/audit_longitudinal_feedback_fixture.py | 85 ++++ tools/run_longitudinal_feedback_experiment.py | 39 ++ 27 files changed, 1917 insertions(+) create mode 100644 docs/longitudinal-feedback-experiment.md create mode 100644 src/neuron_graph_rag/longitudinal_feedback_experiment.py create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback.contamination.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.gold.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.provenance.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.gold.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.provenance.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.gold.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.provenance.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_experiment.manifest.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.gold.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.provenance.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.gold.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.provenance.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.gold.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.provenance.json create mode 100644 tests/test_longitudinal_feedback_experiment.py create mode 100644 tools/audit_longitudinal_feedback_fixture.py create mode 100644 tools/run_longitudinal_feedback_experiment.py diff --git a/README.md b/README.md index a22dd3b..4acaf9d 100644 --- a/README.md +++ b/README.md @@ -243,6 +243,8 @@ Trace-credited feedback adaptationは、relation traceの`record_success`が後 独立reproductionはprior feedback-adaptation resultを選択入力にせず、新規D1 development / holdout splitで同じtrace-credit claimを検証します。relation pathはruntime fieldを除き、`source_id`、`target_id`、`edge_type`だけへ射影してgoldと比較します。prior fixtureは識別子だけのcontamination auditに使い、prior goldとresultは読みません。詳細は[Trace-credited feedback adaptation reproduction experiment](docs/feedback-adaptation-reproduction-experiment.md)を参照してください。 +Longitudinal multi-corpus feedback adaptation fixes `h=0,1,2,3` across three disjoint clusters in each split. Control records the same feedback schedule without edge mutation, while treatment reinforces only credited relation paths. Headroom clusters require strict final-horizon aggregate MRR improvement; ceiling clusters require non-regression only. Earlier observed feedback-adaptation results are never selection inputs. See [Longitudinal multi-corpus feedback adaptation experiment](docs/longitudinal-feedback-experiment.md). + ## Explanation model 各 `SearchHit` は次の情報を保持します。 diff --git a/docs/longitudinal-feedback-experiment.md b/docs/longitudinal-feedback-experiment.md new file mode 100644 index 0000000..f99a68c --- /dev/null +++ b/docs/longitudinal-feedback-experiment.md @@ -0,0 +1,44 @@ +# Longitudinal multi-corpus feedback adaptation experiment + +## Purpose + +This frozen experiment tests whether trace-credited feedback remains directionally useful after multiple success events across independently constructed corpus clusters. It compares a no-mutation control with trace-credited treatment at fixed horizons `h=0,1,2,3`. + +It does not reuse, rerun, modify, aggregate, or use observed results from the earlier feedback-adaptation experiments as selection inputs. Their fixture identifiers are read only by the contamination audit; their gold and observed result artifacts are not loaded. + +## Frozen cohorts and provenance + +- Manifest: `tests/fixtures/d1_liplus_longitudinal_feedback_experiment.manifest.json` +- Each development and holdout split contains three disjoint clusters: two headroom clusters and one ceiling cluster. +- Every cluster has a separately hashed fixture, gold schedule, and provenance record. The records identify these as deterministic in-repository constructed corpora, with no network or D1 access and zero writes. +- The audit compares all six clusters and both earlier feedback-adaptation fixture families by document path, node ID, source URL, normalized query, endpoint, and complete edge identity. It reads prior fixture identifiers only. + +## Schedule and path identity + +Every arm shares the same corpus, configuration, limit, timestamped feedback schedule, and scoring schedule. At horizons one through three, both arms record the same successful relation feedback event. The control records it without an edge mutation; treatment uses only the credited relation trace path. + +Relation path checks retain raw observed steps for audit but compare only the projected identity: + +```json +{"source_id":"...","target_id":"...","edge_type":"..."} +``` + +Runtime fields cannot affect identity. Each horizon also checks a direct lexical case and a directional-negative case. + +## Ceiling-aware gate and one-shot execution + +For headroom clusters, the control relation MRR at `h=0` must be below `1.0`; treatment must strictly improve final-horizon aggregate MRR and must not regress in any headroom cluster. For ceiling clusters, the `h=0` control relation MRR must equal `1.0`, and treatment requires non-regression only. A ceiling pass records that no additional rank improvement is evidenced; it never authorizes default changes, generalization, or production adoption. + +After the result-free freeze is pushed, development is allowed once at its registered output path: + +```powershell +python tools/run_longitudinal_feedback_experiment.py development ` + --manifest tests/fixtures/d1_liplus_longitudinal_feedback_experiment.manifest.json ` + --output tests/fixtures/d1_liplus_longitudinal_feedback_experiment.development.result.json +``` + +Only a passing development result allows one holdout execution at its registered output path. The runner rejects any other output path and refuses to overwrite an observed result. Once observation begins, fixture, gold, provenance, audit, manifest, implementation, tests, documentation, result aggregation, and reruns are forbidden. + +## Interpretation boundary + +Even a passing frozen result supports only longitudinal relation retrieval adaptation on these six small constructed clusters. It does not evaluate agent task success, tool calls, tokens, latency, autonomous-feedback correctness, a production default, or general corpora. diff --git a/docs/requirements.md b/docs/requirements.md index 78014ed..cc3df02 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -94,6 +94,7 @@ 76. feedback adaptationのdevelopment / holdout、feedback event、score query、expected path、gate、registered run count、hash、contamination audit、exclusive outputを結果観測前に固定し、development全gate通過時だけholdoutを一度開く。 77. feedback adaptation reproductionはprior resultを選択入力にせず、prior fixtureとの識別子だけのcontamination auditを行う。raw relation stepは比較前に`source_id`、`target_id`、`edge_type`だけへ射影し、runtime fieldを含むsynthetic testでpath identityをfreeze前に検証する。 78. 新規 feedback-adaptation experiment の primary relation gate は、baseline relation MRR が 1.0 未満ならtreatmentのstrict improvement、baseline relation MRR が 1.0 ならtreatmentのnon-regressionを要求する。いずれも endpoint/type projected path、direct lexical / directional-negative controls、credited-only mutation、deterministic replay、contamination、immutable output を含む全 safety gate を必須とする。ceiling case のnon-regression pass は追加の順位改善を示せないことを記録するだけで、generalization、default変更、production採用を許可しない。 +79. longitudinal feedback adaptation experiment は development / holdout の各3以上の相互disjoint corpus cluster で `h=0,1,2,3` を固定し、同一 feedback schedule の no-mutation control と credited-relation-path-only treatment を比較する。headroom は final-horizon aggregate MRR の strict improvement と cluster non-regression、ceiling は non-regression を要求し、endpoint/type-only path、controls、deterministic replay、contamination、exclusive immutable output を全horizonで必須とする。過去 feedback-adaptation result は selection input にせず、prior fixture identifier のみを contamination audit に使用する。 75. v3 implementation、prompt、manifest、query override、schema、集約、path audit、hash規則、gate、stop rule、testsをresult-free commitでpushした後、development stage / 4 case packet / 12 responses / resultを各一度だけ生成する。 76. development全12 gate通過時だけholdout stageを一度生成し、異なるfresh 12 judgesで同じgateを評価する。packet、response、resultの上書き、観測後の規則変更、実LLM品質値のCI再生成を拒否する。 @@ -125,3 +126,4 @@ - [Node-first blind selection experiment](node-first-blind-selection-experiment.md) がsingle-case invocation、node-majority、channel provenance分離、v1 / v2不変監査、conditional holdoutを定義する。 - [Trace-credited feedback adaptation experiment](feedback-adaptation-experiment.md) がcontrol / treatmentの因果比較、result-free freeze、conditional holdoutを定義する。 - [Trace-credited feedback adaptation reproduction experiment](feedback-adaptation-reproduction-experiment.md) が新規D1 split、prior-result非参照、endpoint/type-only path projection、conditional holdoutを定義する。 +- [Longitudinal multi-corpus feedback adaptation experiment](longitudinal-feedback-experiment.md) が固定horizon、複数cluster、ceiling-aware gate、one-shot conditional holdoutを定義する。 diff --git a/src/neuron_graph_rag/longitudinal_feedback_experiment.py b/src/neuron_graph_rag/longitudinal_feedback_experiment.py new file mode 100644 index 0000000..e772895 --- /dev/null +++ b/src/neuron_graph_rag/longitudinal_feedback_experiment.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from typing import Any + +from .d1_fixture import load_fixture, read_fixture +from .engine import EngineConfig, NeuronGraphRAG + + +LONGITUDINAL_FEEDBACK_SCHEMA_VERSION = 1 +HORIZONS = (0, 1, 2, 3) +PATH_IDENTITY_FIELDS = ("source_id", "target_id", "edge_type") + + +def project_relation_path(steps: list[dict[str, Any]]) -> list[dict[str, str]]: + """Discard runtime-only relation-step fields before identity comparison.""" + return [{field: str(step[field]) for field in PATH_IDENTITY_FIELDS} for step in steps] + + +def read_longitudinal_feedback_manifest(path: str | Path) -> dict[str, Any]: + manifest_path = Path(path) + manifest = _read_json(manifest_path) + if manifest.get("schema_version") != LONGITUDINAL_FEEDBACK_SCHEMA_VERSION: + raise ValueError("Unsupported longitudinal feedback schema version") + if manifest.get("candidate_id") != "trace-credited-longitudinal-feedback": + raise ValueError("Longitudinal feedback candidate is not frozen") + base = manifest_path.parent + seen_clusters: set[str] = set() + for stage in ("development", "holdout"): + clusters = manifest.get(stage, {}).get("clusters") + if not isinstance(clusters, list) or len(clusters) < 3: + raise ValueError(f"{stage} must freeze at least three corpus clusters") + cohorts = {str(cluster.get("cohort")) for cluster in clusters} + if not {"headroom", "ceiling"} <= cohorts: + raise ValueError(f"{stage} must contain headroom and ceiling cohorts") + for cluster in clusters: + cluster_id = str(cluster.get("cluster_id")) + if not cluster_id or cluster_id in seen_clusters: + raise ValueError("Cluster identifiers must be globally disjoint") + seen_clusters.add(cluster_id) + for field in ("fixture", "gold", "provenance"): + artifact = base / str(cluster.get(field, "")) + if _canonical_sha256(artifact) != cluster.get(f"{field}_sha256"): + raise ValueError(f"Frozen {stage} {cluster_id} {field} hash mismatch") + _validate_cluster( + cluster_id, + str(cluster.get("cohort")), + base / str(cluster["fixture"]), + base / str(cluster["gold"]), + base / str(cluster["provenance"]), + ) + audit = manifest.get("contamination_audit", {}) + audit_path = base / str(audit.get("artifact", "")) + if _canonical_sha256(audit_path) != audit.get("artifact_sha256"): + raise ValueError("Frozen longitudinal contamination audit hash mismatch") + if not _read_json(audit_path).get("passed"): + raise ValueError("Frozen longitudinal contamination audit did not pass") + return manifest + + +def run_longitudinal_feedback_development(path: str | Path) -> dict[str, Any]: + manifest_path = Path(path) + manifest = read_longitudinal_feedback_manifest(manifest_path) + result = _evaluate_split(manifest_path, manifest, "development") + passed = all(result["gate"].values()) + return { + "schema_version": LONGITUDINAL_FEEDBACK_SCHEMA_VERSION, + "experiment_id": manifest["experiment_id"], + "stage": "development", + "manifest_sha256": _canonical_sha256(manifest_path), + **result, + "gate_passed": passed, + "holdout_status": "not_opened_candidate_selected" if passed else "not_opened_no_candidate", + } + + +def run_longitudinal_feedback_holdout( + path: str | Path, development_result_path: str | Path +) -> dict[str, Any]: + manifest_path = Path(path) + manifest = read_longitudinal_feedback_manifest(manifest_path) + development_path = Path(development_result_path) + development = _read_json(development_path) + if not development.get("gate_passed"): + raise ValueError("Stop rule forbids opening longitudinal feedback holdout") + expected = manifest_path.parent / str(manifest["outputs"]["development"]) + if development_path.resolve() != expected.resolve(): + raise ValueError("Holdout requires the registered development result path") + if development.get("manifest_sha256") != _canonical_sha256(manifest_path): + raise ValueError("Development result does not match frozen longitudinal manifest") + result = _evaluate_split(manifest_path, manifest, "holdout") + passed = all(result["gate"].values()) + return { + "schema_version": LONGITUDINAL_FEEDBACK_SCHEMA_VERSION, + "experiment_id": manifest["experiment_id"], + "stage": "holdout", + "manifest_sha256": _canonical_sha256(manifest_path), + "development_result_sha256": _canonical_sha256(development_path), + "holdout_open_count": 1, + **result, + "decision": { + "adopted": False, + "reason": "frozen_longitudinal_gate_passed_no_default_or_generalization_claim" + if passed + else "longitudinal_feedback_failed_frozen_holdout_gate", + }, + } + + +def write_longitudinal_feedback_result(path: str | Path, result: dict[str, Any]) -> None: + Path(path).write_text( + json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + newline="\n", + ) + + +def _evaluate_split(path: Path, manifest: dict[str, Any], stage: str) -> dict[str, Any]: + base = path.parent + config, limit = _experiment_config(manifest) + clusters: list[dict[str, Any]] = [] + for cluster in manifest[stage]["clusters"]: + fixture = base / str(cluster["fixture"]) + gold = _read_json(base / str(cluster["gold"])) + control = _run_cluster(fixture, gold, config, limit, mutate=False) + treatment = _run_cluster(fixture, gold, config, limit, mutate=True) + replay = _run_cluster(fixture, gold, config, limit, mutate=True) + clusters.append( + { + "cluster_id": cluster["cluster_id"], + "cohort": cluster["cohort"], + "inputs": {field: cluster[f"{field}_sha256"] for field in ("fixture", "gold", "provenance")}, + "control": control, + "treatment": treatment, + "replay": replay, + } + ) + headroom = [cluster for cluster in clusters if cluster["cohort"] == "headroom"] + ceiling = [cluster for cluster in clusters if cluster["cohort"] == "ceiling"] + metrics = { + "horizon_relation_mrr": { + str(horizon): { + "control": _aggregate_mrr(clusters, "control", horizon), + "treatment": _aggregate_mrr(clusters, "treatment", horizon), + "headroom_control": _aggregate_mrr(headroom, "control", horizon), + "headroom_treatment": _aggregate_mrr(headroom, "treatment", horizon), + "ceiling_control": _aggregate_mrr(ceiling, "control", horizon), + "ceiling_treatment": _aggregate_mrr(ceiling, "treatment", horizon), + } + for horizon in HORIZONS + } + } + final = metrics["horizon_relation_mrr"]["3"] + gate = { + "fixed_horizons_present": all(_has_horizons(cluster["control"]) and _has_horizons(cluster["treatment"]) for cluster in clusters), + "headroom_h0_control_is_below_ceiling": all(_relation_mrr(cluster["control"], 0) < 1.0 for cluster in headroom), + "ceiling_h0_control_equals_ceiling": all(_relation_mrr(cluster["control"], 0) == 1.0 for cluster in ceiling), + "headroom_final_aggregate_mrr_strictly_improves": final["headroom_treatment"] > final["headroom_control"], + "headroom_clusters_do_not_regress": all( + _relation_mrr(cluster["treatment"], 3) >= _relation_mrr(cluster["control"], 3) for cluster in headroom + ), + "ceiling_clusters_do_not_regress": all( + _relation_mrr(cluster["treatment"], 3) >= _relation_mrr(cluster["control"], 3) for cluster in ceiling + ), + "projected_paths_match_at_every_horizon": all(_paths_match(cluster["treatment"]) for cluster in clusters), + "direct_lexical_controls_do_not_regress": all(_controls_do_not_regress(cluster, "direct_lexical") for cluster in clusters), + "directional_negative_controls_remain_empty": all(_negative_controls_remain_empty(cluster) for cluster in clusters), + "credited_only_mutation_and_control_no_mutation": all( + cluster["treatment"]["feedback"]["credited_only"] and cluster["control"]["feedback"]["no_edge_mutation"] + for cluster in clusters + ), + "same_schedule_replay_is_deterministic": all(cluster["treatment"] == cluster["replay"] for cluster in clusters), + "contamination_audit_passed": True, + } + return { + "clusters": clusters, + "metrics": metrics, + "ceiling_policy": { + "headroom": "strict final-horizon aggregate MRR improvement with per-cluster non-regression", + "ceiling": "non-regression only; a pass does not evidence an additional rank improvement", + "ceiling_rank_improvement_observable": any( + _relation_mrr(cluster["treatment"], 3) > _relation_mrr(cluster["control"], 3) for cluster in ceiling + ), + }, + "gate": gate, + } + + +def _run_cluster( + fixture_path: Path, gold: dict[str, Any], config: EngineConfig, limit: int, *, mutate: bool +) -> dict[str, Any]: + schedule = {int(event["horizon"]): event for event in gold["feedback_schedule"]} + credited: list[dict[str, Any]] = [] + with NeuronGraphRAG(config=config) as engine: + load_fixture(engine, fixture_path) + before = _edge_snapshot(engine) + horizons: list[dict[str, Any]] = [] + for horizon in HORIZONS: + if horizon: + event = schedule[horizon] + channels = engine.search_channels(event["query"], limit=limit, now=event["now"]) + if event["channel"] != "relation" or event["used_node_id"] not in {hit.node.node_id for hit in channels.relation.hits}: + raise ValueError("Frozen feedback event was not retrieved by its relation trace") + if mutate: + receipt = engine.record_success( + channels.relation.trace_id, [event["used_node_id"]], now=float(event["now"]) + 0.1 + ) + credited.extend(_reinforced(edge) for edge in receipt.reinforced_edges) + else: + engine.store.apply_success_feedback( + f"control-{horizon}", channels.relation.trace_id, float(event["now"]) + 0.1, + [event["used_node_id"]], (), + ) + horizons.append({"horizon": horizon, "cases": [_score_case(engine, case, limit) for case in gold["scoring_cases"]]}) + after = _edge_snapshot(engine) + changed = _edge_changes(before, after) + credited_keys = {(edge["source_id"], edge["target_id"], edge["edge_type"]) for edge in credited} + return { + "horizons": horizons, + "feedback": { + "recorded": True, + "feedback_count": len(schedule), + "credited_edges": credited, + "edge_changes": changed, + "no_edge_mutation": not changed, + "credited_only": bool(credited) and all( + (edge["source_id"], edge["target_id"], edge["edge_type"]) in credited_keys for edge in changed + ), + }, + } + + +def _score_case(engine: NeuronGraphRAG, case: dict[str, Any], limit: int) -> dict[str, Any]: + channels = engine.search_channels(case["query"], limit=limit, now=case["now"]) + expected = str(case["expected_node_id"]) + lexical_hits = [hit.node.node_id for hit in channels.lexical.hits] + relation_hits = [hit.node.node_id for hit in channels.relation.hits] + observed_paths = [ + path["steps"] + for hit in channels.relation.hits + if hit.node.node_id == expected + for path in hit.explain()["paths"] + ] + projected_paths = [project_relation_path(path) for path in observed_paths] + expected_path = case.get("expected_path", []) + return { + "id": case["id"], + "role": case["role"], + "expected_node_id": expected, + "lexical_rank": _rank(lexical_hits, expected, limit), + "relation_rank": _rank(relation_hits, expected, limit), + "relation_found": expected in relation_hits, + "relation_hits": relation_hits, + "path_matched": any(path == expected_path for path in projected_paths) + if expected_path + else bool(case.get("expected_relation_empty")) and not relation_hits, + "observed_paths": observed_paths, + "projected_observed_paths": projected_paths, + } + + +def _validate_cluster(cluster_id: str, cohort: str, fixture_path: Path, gold_path: Path, provenance_path: Path) -> None: + fixture = read_fixture(fixture_path) + gold = _read_json(gold_path) + provenance = _read_json(provenance_path) + if gold.get("cluster_id") != cluster_id or provenance.get("cluster_id") != cluster_id: + raise ValueError("Cluster artifact identities do not agree") + if cohort not in {"headroom", "ceiling"} or gold.get("cohort") != cohort: + raise ValueError("Cluster cohort is invalid") + nodes = {str(node["node_id"]) for node in fixture["nodes"]} + edges = {(str(edge["source_id"]), str(edge["target_id"]), str(edge["edge_type"])) for edge in fixture["edges"]} + schedule = gold.get("feedback_schedule") + if not isinstance(schedule, list) or [event.get("horizon") for event in schedule] != [1, 2, 3]: + raise ValueError("Feedback schedule must freeze horizons one through three") + cases = gold.get("scoring_cases") + if not isinstance(cases, list) or {case.get("role") for case in cases} != {"relation", "direct_lexical", "directional_negative"}: + raise ValueError("Gold must freeze relation and both controls") + feedback_queries = {str(event.get("query")) for event in schedule} + if feedback_queries & {str(case.get("query")) for case in cases}: + raise ValueError("Feedback queries must be separate from scoring queries") + for event in schedule: + if event.get("channel") != "relation" or str(event.get("used_node_id")) not in nodes: + raise ValueError("Feedback event is invalid") + for case in cases: + if str(case.get("expected_node_id")) not in nodes or not str(case.get("source_url", "")).startswith("https://github.com/"): + raise ValueError("Gold case target or source URL is invalid") + for step in case.get("expected_path", []): + if set(step) != set(PATH_IDENTITY_FIELDS): + raise ValueError("Gold path identity must contain endpoint and edge type only") + if tuple(str(step[field]) for field in PATH_IDENTITY_FIELDS) not in edges: + raise ValueError("Expected relation path is outside fixture") + + +def _experiment_config(manifest: dict[str, Any]) -> tuple[EngineConfig, int]: + values = dict(manifest["shared_config"]) + return EngineConfig(**{key: value for key, value in values.items() if key != "limit"}), int(values["limit"]) + + +def _aggregate_mrr(clusters: list[dict[str, Any]], arm: str, horizon: int) -> float: + ranks = [ + case["relation_rank"] + for cluster in clusters + for row in cluster[arm]["horizons"] + if row["horizon"] == horizon + for case in row["cases"] + if case["role"] == "relation" + ] + return _mrr(ranks) + + +def _relation_mrr(group: dict[str, Any], horizon: int) -> float: + return _mrr([case["relation_rank"] for row in group["horizons"] if row["horizon"] == horizon for case in row["cases"] if case["role"] == "relation"]) + + +def _has_horizons(group: dict[str, Any]) -> bool: + return [row["horizon"] for row in group["horizons"]] == list(HORIZONS) + + +def _paths_match(group: dict[str, Any]) -> bool: + return all(case["path_matched"] for row in group["horizons"] for case in row["cases"] if case["role"] == "relation") + + +def _controls_do_not_regress(cluster: dict[str, Any], role: str) -> bool: + for left, right in zip(cluster["control"]["horizons"], cluster["treatment"]["horizons"], strict=True): + left_case = next(case for case in left["cases"] if case["role"] == role) + right_case = next(case for case in right["cases"] if case["role"] == role) + if right_case["lexical_rank"] > left_case["lexical_rank"]: + return False + return True + + +def _negative_controls_remain_empty(cluster: dict[str, Any]) -> bool: + return all( + case["relation_hits"] == [] + for arm in ("control", "treatment") + for row in cluster[arm]["horizons"] + for case in row["cases"] + if case["role"] == "directional_negative" + ) + + +def _rank(hits: list[str], node_id: str, limit: int) -> int: + return next((index for index, value in enumerate(hits, start=1) if value == node_id), limit + 1) + + +def _mrr(ranks: list[int]) -> float: + return sum(1.0 / rank for rank in ranks) / len(ranks) + + +def _edge_snapshot(engine: NeuronGraphRAG) -> dict[tuple[str, str, str], tuple[float, int]]: + return {(edge.source_id, edge.target_id, edge.edge_type): (edge.weight, edge.reinforced_count) for edge in engine.store.list_edges()} + + +def _edge_changes(before: dict[tuple[str, str, str], tuple[float, int]], after: dict[tuple[str, str, str], tuple[float, int]]) -> list[dict[str, Any]]: + return [{"source_id": key[0], "target_id": key[1], "edge_type": key[2], "before_weight": before[key][0], "after_weight": after[key][0], "before_reinforced_count": before[key][1], "after_reinforced_count": after[key][1]} for key in sorted(before) if before[key] != after[key]] + + +def _reinforced(edge: Any) -> dict[str, Any]: + return {"source_id": edge.source_id, "target_id": edge.target_id, "edge_type": edge.edge_type, "old_weight": edge.old_weight, "new_weight": edge.new_weight} + + +def _read_json(path: Path) -> dict[str, Any]: + with path.open(encoding="utf-8") as stream: + value = json.load(stream) + if not isinstance(value, dict): + raise ValueError(f"Expected JSON object: {path}") + return value + + +def _canonical_sha256(path: str | Path) -> str: + return "sha256:" + hashlib.sha256((json.dumps(_read_json(Path(path)), ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8")).hexdigest() diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback.contamination.json b/tests/fixtures/d1_liplus_longitudinal_feedback.contamination.json new file mode 100644 index 0000000..0e13e19 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback.contamination.json @@ -0,0 +1,371 @@ +{ + "checks": { + "development:lf-development-ceiling_vs_development:lf-development-headroom-a": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_development:lf-development-headroom-b": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_holdout:lf-holdout-ceiling": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_holdout:lf-holdout-headroom-a": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_holdout:lf-holdout-headroom-b": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_prior_1": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_prior_2": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_prior_3": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-ceiling_vs_prior_4": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_development:lf-development-headroom-b": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_holdout:lf-holdout-ceiling": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_holdout:lf-holdout-headroom-a": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_holdout:lf-holdout-headroom-b": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_prior_1": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_prior_2": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_prior_3": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-a_vs_prior_4": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-b_vs_holdout:lf-holdout-ceiling": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-b_vs_holdout:lf-holdout-headroom-a": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-b_vs_holdout:lf-holdout-headroom-b": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-b_vs_prior_1": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-b_vs_prior_2": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-b_vs_prior_3": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "development:lf-development-headroom-b_vs_prior_4": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-ceiling_vs_holdout:lf-holdout-headroom-a": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-ceiling_vs_holdout:lf-holdout-headroom-b": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-ceiling_vs_prior_1": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-ceiling_vs_prior_2": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-ceiling_vs_prior_3": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-ceiling_vs_prior_4": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-a_vs_holdout:lf-holdout-headroom-b": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-a_vs_prior_1": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-a_vs_prior_2": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-a_vs_prior_3": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-a_vs_prior_4": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-b_vs_prior_1": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-b_vs_prior_2": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-b_vs_prior_3": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + }, + "holdout:lf-holdout-headroom-b_vs_prior_4": { + "doc_paths": [], + "edge_identities": [], + "endpoints": [], + "node_ids": [], + "queries": [], + "source_urls": [] + } + }, + "inputs": { + "clusters": [ + { + "cluster": "development:lf-development-headroom-a", + "fixture_sha256": "sha256:ff4cccd92c87c2f3f8caf75d10e2f9284c8e1e03ba3967a15c79794697a7c260", + "gold_sha256": "sha256:dd7eb035ecb28d72028441035812113475c695089bedea43ceeda90c047196c1" + }, + { + "cluster": "development:lf-development-headroom-b", + "fixture_sha256": "sha256:c525765e7a54f57ea4b16ab2ff4016e64f5340fe4bbc89846081f2fbe5da4cb0", + "gold_sha256": "sha256:fc425e8fdbc566b5b7c17899793f0f018e2d87e716b6ea50ed7512d260c8ea9c" + }, + { + "cluster": "development:lf-development-ceiling", + "fixture_sha256": "sha256:4436560b9a8c88d3474fcbc4401dd641a09c04aeb8f6550d8c27868bf2c7a56b", + "gold_sha256": "sha256:95faec8c3837ad05155c0e1fe0419e3f8a70a20659fa6fd0674418c5874f1527" + }, + { + "cluster": "holdout:lf-holdout-headroom-a", + "fixture_sha256": "sha256:68d3c0a516bfed8467e63243ecd091c7934222df6ce16d8cc81a5d73e50498b5", + "gold_sha256": "sha256:8874747f2e9090b674e24cdfd9fab5f4f8605562fb2f057db26aa84e03e19cc7" + }, + { + "cluster": "holdout:lf-holdout-headroom-b", + "fixture_sha256": "sha256:ae3f7e3b63f5e01e855d573e169c1aa703ad1a2058be8fb250564937977503ea", + "gold_sha256": "sha256:bf40d4b51fb693956543e3ee2ea738a59be42405ca29ad367b9fcc7b76e3fd02" + }, + { + "cluster": "holdout:lf-holdout-ceiling", + "fixture_sha256": "sha256:634aac3c901cdbe64f56abd89d7a66d6e7ab70b2c3daf536e26582c7341c7c07", + "gold_sha256": "sha256:f92571b85abc849a87e1c29dd67c4a5e35b032b7179567895971c530bef8efe7" + } + ], + "prior_fixtures": [ + { + "fixture": "d1_liplus_feedback_adaptation_development.json", + "fixture_sha256": "sha256:65f9993d3de6adae368daded3525bb79bf20eae613efde17174ba0685b9bc46c" + }, + { + "fixture": "d1_liplus_feedback_adaptation_holdout.json", + "fixture_sha256": "sha256:44d743353a70826c3933bc19a759de15e1a3d297cac5b4c448af686fdc33d919" + }, + { + "fixture": "d1_liplus_feedback_adaptation_reproduction_development.json", + "fixture_sha256": "sha256:09a89a5bc283b2f29a73889ffbe2780c264b2e526763f3744b77139431943c58" + }, + { + "fixture": "d1_liplus_feedback_adaptation_reproduction_holdout.json", + "fixture_sha256": "sha256:9154fb635ee7904c4a5fc7aaa2c1e67ecd887dfc1fc525c336661384efc42a25" + } + ] + }, + "passed": true, + "prior_usage": "fixture identifiers only; prior gold and observed result artifacts are not loaded", + "schema_version": 1 +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.gold.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.gold.json new file mode 100644 index 0000000..8c45517 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.gold.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "cluster_id": "lf-development-ceiling", + "cohort": "ceiling", + "feedback_schedule": [ + { + "horizon": 1, + "query": "confirm successful lf-development-ceiling trace", + "used_node_id": "lf-development-ceiling-b-target", + "channel": "relation", + "now": 101 + }, + { + "horizon": 2, + "query": "confirm successful lf-development-ceiling trace", + "used_node_id": "lf-development-ceiling-b-target", + "channel": "relation", + "now": 102 + }, + { + "horizon": 3, + "query": "confirm successful lf-development-ceiling trace", + "used_node_id": "lf-development-ceiling-b-target", + "channel": "relation", + "now": 103 + } + ], + "scoring_cases": [ + { + "id": "lf-development-ceiling-relation", + "role": "relation", + "query": "longitudinal lf-development-ceiling seed relation", + "expected_node_id": "lf-development-ceiling-b-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-target", + "expected_path": [ + { + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target", + "edge_type": "supports" + } + ], + "now": 100 + }, + { + "id": "lf-development-ceiling-direct", + "role": "direct_lexical", + "query": "direct lexical lf-development-ceiling target answer", + "expected_node_id": "lf-development-ceiling-b-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-target", + "expected_relation_empty": true, + "now": 100 + }, + { + "id": "lf-development-ceiling-negative", + "role": "directional_negative", + "query": "impossible reverse lf-development-ceiling direction", + "expected_node_id": "lf-development-ceiling-source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-source", + "expected_relation_empty": true, + "now": 100 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.json new file mode 100644 index 0000000..9c7e684 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "nodes": [ + { + "node_id": "lf-development-ceiling-source", + "text": "Longitudinal corpus lf-development-ceiling seed relation scoring confirm successful trace.", + "metadata": { + "doc_path": "longitudinal-feedback/development/ceiling/source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-source", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-development-ceiling-z-rival", + "text": "Longitudinal corpus lf-development-ceiling rival relation destination.", + "metadata": { + "doc_path": "longitudinal-feedback/development/ceiling/rival", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-rival", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-development-ceiling-b-target", + "text": "Longitudinal corpus lf-development-ceiling target direct lexical answer.", + "metadata": { + "doc_path": "longitudinal-feedback/development/ceiling/target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-target", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + } + ], + "edges": [ + { + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-z-rival", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + }, + { + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.provenance.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.provenance.json new file mode 100644 index 0000000..e0a5b97 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_ceiling.provenance.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "cluster_id": "lf-development-ceiling", + "cohort": "ceiling", + "build_method": "deterministic in-repository corpus fixture", + "source_access": "no network or D1 access", + "read_only": true, + "writes": 0, + "acquired_at": "2026-08-09T00:00:00Z", + "document_count": 3, + "edge_count": 2, + "source_urls": [ + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-source", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-rival", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-ceiling-target" + ], + "notes": "Result-free constructed corpus; identifiers are unique across every longitudinal cluster and the prior feedback-adaptation fixture families." +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.gold.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.gold.json new file mode 100644 index 0000000..fcab921 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.gold.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "cluster_id": "lf-development-headroom-a", + "cohort": "headroom", + "feedback_schedule": [ + { + "horizon": 1, + "query": "confirm successful lf-development-headroom-a trace", + "used_node_id": "lf-development-headroom-a-z-target", + "channel": "relation", + "now": 101 + }, + { + "horizon": 2, + "query": "confirm successful lf-development-headroom-a trace", + "used_node_id": "lf-development-headroom-a-z-target", + "channel": "relation", + "now": 102 + }, + { + "horizon": 3, + "query": "confirm successful lf-development-headroom-a trace", + "used_node_id": "lf-development-headroom-a-z-target", + "channel": "relation", + "now": 103 + } + ], + "scoring_cases": [ + { + "id": "lf-development-headroom-a-relation", + "role": "relation", + "query": "longitudinal lf-development-headroom-a seed relation", + "expected_node_id": "lf-development-headroom-a-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-target", + "expected_path": [ + { + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target", + "edge_type": "supports" + } + ], + "now": 100 + }, + { + "id": "lf-development-headroom-a-direct", + "role": "direct_lexical", + "query": "direct lexical lf-development-headroom-a target answer", + "expected_node_id": "lf-development-headroom-a-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-target", + "expected_relation_empty": true, + "now": 100 + }, + { + "id": "lf-development-headroom-a-negative", + "role": "directional_negative", + "query": "impossible reverse lf-development-headroom-a direction", + "expected_node_id": "lf-development-headroom-a-source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-source", + "expected_relation_empty": true, + "now": 100 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.json new file mode 100644 index 0000000..d37a02f --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "nodes": [ + { + "node_id": "lf-development-headroom-a-source", + "text": "Longitudinal corpus lf-development-headroom-a seed relation scoring confirm successful trace.", + "metadata": { + "doc_path": "longitudinal-feedback/development/headroom-a/source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-source", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-development-headroom-a-b-rival", + "text": "Longitudinal corpus lf-development-headroom-a rival relation destination.", + "metadata": { + "doc_path": "longitudinal-feedback/development/headroom-a/rival", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-rival", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-development-headroom-a-z-target", + "text": "Longitudinal corpus lf-development-headroom-a target direct lexical answer.", + "metadata": { + "doc_path": "longitudinal-feedback/development/headroom-a/target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-target", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + } + ], + "edges": [ + { + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-b-rival", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + }, + { + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.provenance.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.provenance.json new file mode 100644 index 0000000..af32b47 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-a.provenance.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "cluster_id": "lf-development-headroom-a", + "cohort": "headroom", + "build_method": "deterministic in-repository corpus fixture", + "source_access": "no network or D1 access", + "read_only": true, + "writes": 0, + "acquired_at": "2026-08-09T00:00:00Z", + "document_count": 3, + "edge_count": 2, + "source_urls": [ + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-source", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-rival", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-a-target" + ], + "notes": "Result-free constructed corpus; identifiers are unique across every longitudinal cluster and the prior feedback-adaptation fixture families." +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.gold.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.gold.json new file mode 100644 index 0000000..ce147e8 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.gold.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "cluster_id": "lf-development-headroom-b", + "cohort": "headroom", + "feedback_schedule": [ + { + "horizon": 1, + "query": "confirm successful lf-development-headroom-b trace", + "used_node_id": "lf-development-headroom-b-z-target", + "channel": "relation", + "now": 101 + }, + { + "horizon": 2, + "query": "confirm successful lf-development-headroom-b trace", + "used_node_id": "lf-development-headroom-b-z-target", + "channel": "relation", + "now": 102 + }, + { + "horizon": 3, + "query": "confirm successful lf-development-headroom-b trace", + "used_node_id": "lf-development-headroom-b-z-target", + "channel": "relation", + "now": 103 + } + ], + "scoring_cases": [ + { + "id": "lf-development-headroom-b-relation", + "role": "relation", + "query": "longitudinal lf-development-headroom-b seed relation", + "expected_node_id": "lf-development-headroom-b-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-target", + "expected_path": [ + { + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target", + "edge_type": "supports" + } + ], + "now": 100 + }, + { + "id": "lf-development-headroom-b-direct", + "role": "direct_lexical", + "query": "direct lexical lf-development-headroom-b target answer", + "expected_node_id": "lf-development-headroom-b-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-target", + "expected_relation_empty": true, + "now": 100 + }, + { + "id": "lf-development-headroom-b-negative", + "role": "directional_negative", + "query": "impossible reverse lf-development-headroom-b direction", + "expected_node_id": "lf-development-headroom-b-source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-source", + "expected_relation_empty": true, + "now": 100 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.json new file mode 100644 index 0000000..fbc208b --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "nodes": [ + { + "node_id": "lf-development-headroom-b-source", + "text": "Longitudinal corpus lf-development-headroom-b seed relation scoring confirm successful trace.", + "metadata": { + "doc_path": "longitudinal-feedback/development/headroom-b/source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-source", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-development-headroom-b-b-rival", + "text": "Longitudinal corpus lf-development-headroom-b rival relation destination.", + "metadata": { + "doc_path": "longitudinal-feedback/development/headroom-b/rival", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-rival", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-development-headroom-b-z-target", + "text": "Longitudinal corpus lf-development-headroom-b target direct lexical answer.", + "metadata": { + "doc_path": "longitudinal-feedback/development/headroom-b/target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-target", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + } + ], + "edges": [ + { + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-b-rival", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + }, + { + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.provenance.json b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.provenance.json new file mode 100644 index 0000000..1e8e6c0 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_development_headroom-b.provenance.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "cluster_id": "lf-development-headroom-b", + "cohort": "headroom", + "build_method": "deterministic in-repository corpus fixture", + "source_access": "no network or D1 access", + "read_only": true, + "writes": 0, + "acquired_at": "2026-08-09T00:00:00Z", + "document_count": 3, + "edge_count": 2, + "source_urls": [ + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-source", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-rival", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-development-headroom-b-target" + ], + "notes": "Result-free constructed corpus; identifiers are unique across every longitudinal cluster and the prior feedback-adaptation fixture families." +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.manifest.json b/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.manifest.json new file mode 100644 index 0000000..8ecab7b --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.manifest.json @@ -0,0 +1,139 @@ +{ + "schema_version": 1, + "experiment_id": "multi-corpus-trace-credited-longitudinal-feedback-v1", + "candidate_id": "trace-credited-longitudinal-feedback", + "shared_config": { + "activation_strategy": "anchored_local_competition", + "use_dense_retrieval": false, + "dense_weight": 0, + "entry_weight": 0, + "graph_weight": 1, + "graph_normalization": "none", + "hop_decay": 1, + "inhibition_ratio": 0, + "max_hops": 1, + "seed_count": 1, + "feedback_learning_rate": 0.2, + "sparse_weight": 1, + "limit": 2 + }, + "development": { + "clusters": [ + { + "cluster_id": "lf-development-headroom-a", + "cohort": "headroom", + "fixture": "d1_liplus_longitudinal_feedback_development_headroom-a.json", + "fixture_sha256": "sha256:ff4cccd92c87c2f3f8caf75d10e2f9284c8e1e03ba3967a15c79794697a7c260", + "gold": "d1_liplus_longitudinal_feedback_development_headroom-a.gold.json", + "gold_sha256": "sha256:dd7eb035ecb28d72028441035812113475c695089bedea43ceeda90c047196c1", + "provenance": "d1_liplus_longitudinal_feedback_development_headroom-a.provenance.json", + "provenance_sha256": "sha256:92259a3c4a7953f67b4b632c02537173d19970acc13018564371d449e3d6b86a" + }, + { + "cluster_id": "lf-development-headroom-b", + "cohort": "headroom", + "fixture": "d1_liplus_longitudinal_feedback_development_headroom-b.json", + "fixture_sha256": "sha256:c525765e7a54f57ea4b16ab2ff4016e64f5340fe4bbc89846081f2fbe5da4cb0", + "gold": "d1_liplus_longitudinal_feedback_development_headroom-b.gold.json", + "gold_sha256": "sha256:fc425e8fdbc566b5b7c17899793f0f018e2d87e716b6ea50ed7512d260c8ea9c", + "provenance": "d1_liplus_longitudinal_feedback_development_headroom-b.provenance.json", + "provenance_sha256": "sha256:7b156687006c70f3f30183c6e72f7a5e145daef83d26125669e0ab26e9051735" + }, + { + "cluster_id": "lf-development-ceiling", + "cohort": "ceiling", + "fixture": "d1_liplus_longitudinal_feedback_development_ceiling.json", + "fixture_sha256": "sha256:4436560b9a8c88d3474fcbc4401dd641a09c04aeb8f6550d8c27868bf2c7a56b", + "gold": "d1_liplus_longitudinal_feedback_development_ceiling.gold.json", + "gold_sha256": "sha256:95faec8c3837ad05155c0e1fe0419e3f8a70a20659fa6fd0674418c5874f1527", + "provenance": "d1_liplus_longitudinal_feedback_development_ceiling.provenance.json", + "provenance_sha256": "sha256:1fd5cbeb39349f29d3628ca8ff4fec4ac6968e3a6863823705330e588866ac48" + } + ] + }, + "holdout": { + "clusters": [ + { + "cluster_id": "lf-holdout-headroom-a", + "cohort": "headroom", + "fixture": "d1_liplus_longitudinal_feedback_holdout_headroom-a.json", + "fixture_sha256": "sha256:68d3c0a516bfed8467e63243ecd091c7934222df6ce16d8cc81a5d73e50498b5", + "gold": "d1_liplus_longitudinal_feedback_holdout_headroom-a.gold.json", + "gold_sha256": "sha256:8874747f2e9090b674e24cdfd9fab5f4f8605562fb2f057db26aa84e03e19cc7", + "provenance": "d1_liplus_longitudinal_feedback_holdout_headroom-a.provenance.json", + "provenance_sha256": "sha256:15fedd655aa5c482d5dcef89820bf24f7912937a268e447b84e8a93c84ced724" + }, + { + "cluster_id": "lf-holdout-headroom-b", + "cohort": "headroom", + "fixture": "d1_liplus_longitudinal_feedback_holdout_headroom-b.json", + "fixture_sha256": "sha256:ae3f7e3b63f5e01e855d573e169c1aa703ad1a2058be8fb250564937977503ea", + "gold": "d1_liplus_longitudinal_feedback_holdout_headroom-b.gold.json", + "gold_sha256": "sha256:bf40d4b51fb693956543e3ee2ea738a59be42405ca29ad367b9fcc7b76e3fd02", + "provenance": "d1_liplus_longitudinal_feedback_holdout_headroom-b.provenance.json", + "provenance_sha256": "sha256:2bfb8ede3cfdabf4259e6acf816a218a3eac3c0fa4d82a4e0f4ec1d94682b922" + }, + { + "cluster_id": "lf-holdout-ceiling", + "cohort": "ceiling", + "fixture": "d1_liplus_longitudinal_feedback_holdout_ceiling.json", + "fixture_sha256": "sha256:634aac3c901cdbe64f56abd89d7a66d6e7ab70b2c3daf536e26582c7341c7c07", + "gold": "d1_liplus_longitudinal_feedback_holdout_ceiling.gold.json", + "gold_sha256": "sha256:f92571b85abc849a87e1c29dd67c4a5e35b032b7179567895971c530bef8efe7", + "provenance": "d1_liplus_longitudinal_feedback_holdout_ceiling.provenance.json", + "provenance_sha256": "sha256:4cb9fa126fc9943dc2ef36647632b0b175a3fb764b6a82ca10f492b5fcb56b6b" + } + ] + }, + "contamination_audit": { + "artifact": "d1_liplus_longitudinal_feedback.contamination.json", + "artifact_sha256": "sha256:8d183955b3330fa9078e306464372c637964b6e7bd350b1012dc590ac86044d4" + }, + "outputs": { + "development": "d1_liplus_longitudinal_feedback_experiment.development.result.json", + "holdout": "d1_liplus_longitudinal_feedback_experiment.holdout.result.json" + }, + "registered_runs": { + "development": { + "control": 1, + "treatment": 2, + "horizons": [ + 0, + 1, + 2, + 3 + ] + }, + "holdout": { + "conditional_on_development_gate": true, + "open_count": 1, + "control": 1, + "treatment": 2, + "horizons": [ + 0, + 1, + 2, + 3 + ] + } + }, + "gate": [ + "fixed_horizons_present", + "headroom_h0_control_is_below_ceiling", + "ceiling_h0_control_equals_ceiling", + "headroom_final_aggregate_mrr_strictly_improves", + "headroom_clusters_do_not_regress", + "ceiling_clusters_do_not_regress", + "projected_paths_match_at_every_horizon", + "direct_lexical_controls_do_not_regress", + "directional_negative_controls_remain_empty", + "credited_only_mutation_and_control_no_mutation", + "same_schedule_replay_is_deterministic", + "contamination_audit_passed" + ], + "stop_rule": { + "development_failure": "do not open holdout", + "exclusive_output": "only the two registered result paths may be used; existing output is never overwritten", + "post_observation_changes": "fixture, gold, provenance, audit, manifest, implementation, tests, docs, result overwrite, aggregation, and rerun are forbidden" + } +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.gold.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.gold.json new file mode 100644 index 0000000..08e8114 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.gold.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "cluster_id": "lf-holdout-ceiling", + "cohort": "ceiling", + "feedback_schedule": [ + { + "horizon": 1, + "query": "confirm successful lf-holdout-ceiling trace", + "used_node_id": "lf-holdout-ceiling-b-target", + "channel": "relation", + "now": 101 + }, + { + "horizon": 2, + "query": "confirm successful lf-holdout-ceiling trace", + "used_node_id": "lf-holdout-ceiling-b-target", + "channel": "relation", + "now": 102 + }, + { + "horizon": 3, + "query": "confirm successful lf-holdout-ceiling trace", + "used_node_id": "lf-holdout-ceiling-b-target", + "channel": "relation", + "now": 103 + } + ], + "scoring_cases": [ + { + "id": "lf-holdout-ceiling-relation", + "role": "relation", + "query": "longitudinal lf-holdout-ceiling seed relation", + "expected_node_id": "lf-holdout-ceiling-b-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-target", + "expected_path": [ + { + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target", + "edge_type": "supports" + } + ], + "now": 100 + }, + { + "id": "lf-holdout-ceiling-direct", + "role": "direct_lexical", + "query": "direct lexical lf-holdout-ceiling target answer", + "expected_node_id": "lf-holdout-ceiling-b-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-target", + "expected_relation_empty": true, + "now": 100 + }, + { + "id": "lf-holdout-ceiling-negative", + "role": "directional_negative", + "query": "impossible reverse lf-holdout-ceiling direction", + "expected_node_id": "lf-holdout-ceiling-source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-source", + "expected_relation_empty": true, + "now": 100 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.json new file mode 100644 index 0000000..7265d45 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "nodes": [ + { + "node_id": "lf-holdout-ceiling-source", + "text": "Longitudinal corpus lf-holdout-ceiling seed relation scoring confirm successful trace.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/ceiling/source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-source", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-holdout-ceiling-z-rival", + "text": "Longitudinal corpus lf-holdout-ceiling rival relation destination.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/ceiling/rival", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-rival", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-holdout-ceiling-b-target", + "text": "Longitudinal corpus lf-holdout-ceiling target direct lexical answer.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/ceiling/target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-target", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + } + ], + "edges": [ + { + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-z-rival", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + }, + { + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.provenance.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.provenance.json new file mode 100644 index 0000000..33ac3f4 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_ceiling.provenance.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "cluster_id": "lf-holdout-ceiling", + "cohort": "ceiling", + "build_method": "deterministic in-repository corpus fixture", + "source_access": "no network or D1 access", + "read_only": true, + "writes": 0, + "acquired_at": "2026-08-09T00:00:00Z", + "document_count": 3, + "edge_count": 2, + "source_urls": [ + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-source", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-rival", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-ceiling-target" + ], + "notes": "Result-free constructed corpus; identifiers are unique across every longitudinal cluster and the prior feedback-adaptation fixture families." +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.gold.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.gold.json new file mode 100644 index 0000000..15ce9c8 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.gold.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "cluster_id": "lf-holdout-headroom-a", + "cohort": "headroom", + "feedback_schedule": [ + { + "horizon": 1, + "query": "confirm successful lf-holdout-headroom-a trace", + "used_node_id": "lf-holdout-headroom-a-z-target", + "channel": "relation", + "now": 101 + }, + { + "horizon": 2, + "query": "confirm successful lf-holdout-headroom-a trace", + "used_node_id": "lf-holdout-headroom-a-z-target", + "channel": "relation", + "now": 102 + }, + { + "horizon": 3, + "query": "confirm successful lf-holdout-headroom-a trace", + "used_node_id": "lf-holdout-headroom-a-z-target", + "channel": "relation", + "now": 103 + } + ], + "scoring_cases": [ + { + "id": "lf-holdout-headroom-a-relation", + "role": "relation", + "query": "longitudinal lf-holdout-headroom-a seed relation", + "expected_node_id": "lf-holdout-headroom-a-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-target", + "expected_path": [ + { + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target", + "edge_type": "supports" + } + ], + "now": 100 + }, + { + "id": "lf-holdout-headroom-a-direct", + "role": "direct_lexical", + "query": "direct lexical lf-holdout-headroom-a target answer", + "expected_node_id": "lf-holdout-headroom-a-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-target", + "expected_relation_empty": true, + "now": 100 + }, + { + "id": "lf-holdout-headroom-a-negative", + "role": "directional_negative", + "query": "impossible reverse lf-holdout-headroom-a direction", + "expected_node_id": "lf-holdout-headroom-a-source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-source", + "expected_relation_empty": true, + "now": 100 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.json new file mode 100644 index 0000000..12593c6 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "nodes": [ + { + "node_id": "lf-holdout-headroom-a-source", + "text": "Longitudinal corpus lf-holdout-headroom-a seed relation scoring confirm successful trace.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/headroom-a/source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-source", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-holdout-headroom-a-b-rival", + "text": "Longitudinal corpus lf-holdout-headroom-a rival relation destination.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/headroom-a/rival", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-rival", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-holdout-headroom-a-z-target", + "text": "Longitudinal corpus lf-holdout-headroom-a target direct lexical answer.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/headroom-a/target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-target", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + } + ], + "edges": [ + { + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-b-rival", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + }, + { + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.provenance.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.provenance.json new file mode 100644 index 0000000..d66435b --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-a.provenance.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "cluster_id": "lf-holdout-headroom-a", + "cohort": "headroom", + "build_method": "deterministic in-repository corpus fixture", + "source_access": "no network or D1 access", + "read_only": true, + "writes": 0, + "acquired_at": "2026-08-09T00:00:00Z", + "document_count": 3, + "edge_count": 2, + "source_urls": [ + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-source", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-rival", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-a-target" + ], + "notes": "Result-free constructed corpus; identifiers are unique across every longitudinal cluster and the prior feedback-adaptation fixture families." +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.gold.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.gold.json new file mode 100644 index 0000000..feca970 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.gold.json @@ -0,0 +1,63 @@ +{ + "schema_version": 1, + "cluster_id": "lf-holdout-headroom-b", + "cohort": "headroom", + "feedback_schedule": [ + { + "horizon": 1, + "query": "confirm successful lf-holdout-headroom-b trace", + "used_node_id": "lf-holdout-headroom-b-z-target", + "channel": "relation", + "now": 101 + }, + { + "horizon": 2, + "query": "confirm successful lf-holdout-headroom-b trace", + "used_node_id": "lf-holdout-headroom-b-z-target", + "channel": "relation", + "now": 102 + }, + { + "horizon": 3, + "query": "confirm successful lf-holdout-headroom-b trace", + "used_node_id": "lf-holdout-headroom-b-z-target", + "channel": "relation", + "now": 103 + } + ], + "scoring_cases": [ + { + "id": "lf-holdout-headroom-b-relation", + "role": "relation", + "query": "longitudinal lf-holdout-headroom-b seed relation", + "expected_node_id": "lf-holdout-headroom-b-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-target", + "expected_path": [ + { + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target", + "edge_type": "supports" + } + ], + "now": 100 + }, + { + "id": "lf-holdout-headroom-b-direct", + "role": "direct_lexical", + "query": "direct lexical lf-holdout-headroom-b target answer", + "expected_node_id": "lf-holdout-headroom-b-z-target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-target", + "expected_relation_empty": true, + "now": 100 + }, + { + "id": "lf-holdout-headroom-b-negative", + "role": "directional_negative", + "query": "impossible reverse lf-holdout-headroom-b direction", + "expected_node_id": "lf-holdout-headroom-b-source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-source", + "expected_relation_empty": true, + "now": 100 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.json new file mode 100644 index 0000000..844c2c6 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.json @@ -0,0 +1,51 @@ +{ + "schema_version": 1, + "nodes": [ + { + "node_id": "lf-holdout-headroom-b-source", + "text": "Longitudinal corpus lf-holdout-headroom-b seed relation scoring confirm successful trace.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/headroom-b/source", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-source", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-holdout-headroom-b-b-rival", + "text": "Longitudinal corpus lf-holdout-headroom-b rival relation destination.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/headroom-b/rival", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-rival", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + }, + { + "node_id": "lf-holdout-headroom-b-z-target", + "text": "Longitudinal corpus lf-holdout-headroom-b target direct lexical answer.", + "metadata": { + "doc_path": "longitudinal-feedback/holdout/headroom-b/target", + "source_url": "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-target", + "source_table": "constructed_longitudinal_fixture" + }, + "confidence": 1 + } + ], + "edges": [ + { + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-b-rival", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + }, + { + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target", + "edge_type": "supports", + "weight": 1, + "factuality": 1 + } + ] +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.provenance.json b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.provenance.json new file mode 100644 index 0000000..adc8adc --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_holdout_headroom-b.provenance.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "cluster_id": "lf-holdout-headroom-b", + "cohort": "headroom", + "build_method": "deterministic in-repository corpus fixture", + "source_access": "no network or D1 access", + "read_only": true, + "writes": 0, + "acquired_at": "2026-08-09T00:00:00Z", + "document_count": 3, + "edge_count": 2, + "source_urls": [ + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-source", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-rival", + "https://github.com/Liplus-Project/neuron-graph-rag/blob/main/docs/longitudinal-feedback-experiment.md#lf-holdout-headroom-b-target" + ], + "notes": "Result-free constructed corpus; identifiers are unique across every longitudinal cluster and the prior feedback-adaptation fixture families." +} diff --git a/tests/test_longitudinal_feedback_experiment.py b/tests/test_longitudinal_feedback_experiment.py new file mode 100644 index 0000000..6d3feb5 --- /dev/null +++ b/tests/test_longitudinal_feedback_experiment.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from neuron_graph_rag.longitudinal_feedback_experiment import ( + HORIZONS, + PATH_IDENTITY_FIELDS, + _canonical_sha256, + _experiment_config, + project_relation_path, + read_longitudinal_feedback_manifest, + run_longitudinal_feedback_holdout, +) + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURES = ROOT / "tests" / "fixtures" +MANIFEST = FIXTURES / "d1_liplus_longitudinal_feedback_experiment.manifest.json" +DEVELOPMENT_RESULT = FIXTURES / "d1_liplus_longitudinal_feedback_experiment.development.result.json" + + +class LongitudinalFeedbackFreezeTest(unittest.TestCase): + def test_path_identity_projects_away_runtime_fields(self) -> None: + raw_step = {"source_id": "source", "target_id": "target", "edge_type": "supports", "edge_weight": 1.2, "factuality": 0.9, "activation": 0.4, "runtime_trace_id": "ignored"} + projected = project_relation_path([raw_step]) + self.assertEqual(projected, [{"source_id": "source", "target_id": "target", "edge_type": "supports"}]) + self.assertEqual(set(projected[0]), set(PATH_IDENTITY_FIELDS)) + + def test_manifest_freezes_six_disjoint_clusters_and_horizons(self) -> None: + manifest = read_longitudinal_feedback_manifest(MANIFEST) + clusters = [cluster for stage in ("development", "holdout") for cluster in manifest[stage]["clusters"]] + self.assertEqual(len(clusters), 6) + self.assertEqual({cluster["cohort"] for cluster in clusters}, {"headroom", "ceiling"}) + self.assertEqual(manifest["registered_runs"]["development"]["horizons"], list(HORIZONS)) + self.assertTrue(manifest["registered_runs"]["holdout"]["conditional_on_development_gate"]) + audit = json.loads((FIXTURES / manifest["contamination_audit"]["artifact"]).read_text(encoding="utf-8")) + self.assertTrue(audit["passed"]) + self.assertEqual(audit["prior_usage"], "fixture identifiers only; prior gold and observed result artifacts are not loaded") + + def test_registered_runner_refuses_unregistered_or_existing_output_without_evaluation(self) -> None: + environment = os.environ | {"PYTHONPATH": str(ROOT / "src")} + unregistered = subprocess.run([sys.executable, "tools/run_longitudinal_feedback_experiment.py", "development", "--manifest", str(MANIFEST), "--output", "unregistered.json"], cwd=ROOT, env=environment, capture_output=True, text=True, check=False) + self.assertNotEqual(unregistered.returncode, 0) + self.assertIn("Refusing unregistered output path", unregistered.stderr) + if DEVELOPMENT_RESULT.exists(): + existing = subprocess.run([sys.executable, "tools/run_longitudinal_feedback_experiment.py", "development", "--manifest", str(MANIFEST), "--output", str(DEVELOPMENT_RESULT)], cwd=ROOT, env=environment, capture_output=True, text=True, check=False) + self.assertNotEqual(existing.returncode, 0) + self.assertIn("Refusing to overwrite an observed result", existing.stderr) + + def test_failed_development_cannot_open_holdout(self) -> None: + with tempfile.TemporaryDirectory() as directory: + failed = Path(directory) / "development.json" + failed.write_text(json.dumps({"manifest_sha256": _canonical_sha256(MANIFEST), "gate_passed": False}), encoding="utf-8") + with self.assertRaisesRegex(ValueError, "Stop rule forbids opening"): + run_longitudinal_feedback_holdout(MANIFEST, failed) + + def test_limit_is_a_runner_setting_not_an_engine_config_field(self) -> None: + config, limit = _experiment_config(json.loads(MANIFEST.read_text(encoding="utf-8"))) + self.assertEqual(limit, 2) + self.assertEqual(config.seed_count, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/audit_longitudinal_feedback_fixture.py b/tools/audit_longitudinal_feedback_fixture.py new file mode 100644 index 0000000..2381835 --- /dev/null +++ b/tools/audit_longitudinal_feedback_fixture.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + + +def _read(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"Expected JSON object: {path}") + return value + + +def _sha(path: Path) -> str: + return "sha256:" + hashlib.sha256( + (json.dumps(_read(path), ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8") + ).hexdigest() + + +def _identifiers(fixture: Path, gold: Path | None = None) -> dict[str, set[str]]: + corpus = _read(fixture) + values = { + "doc_paths": {str(node.get("metadata", {}).get("doc_path", "")) for node in corpus["nodes"]}, + "node_ids": {str(node["node_id"]) for node in corpus["nodes"]}, + "source_urls": {str(node.get("metadata", {}).get("source_url", "")) for node in corpus["nodes"]}, + "endpoints": {str(value) for edge in corpus["edges"] for value in (edge["source_id"], edge["target_id"])}, + "edge_identities": {"\u0000".join((str(edge["source_id"]), str(edge["target_id"]), str(edge["edge_type"]))) for edge in corpus["edges"]}, + "queries": set(), + } + if gold is not None: + schedule = _read(gold) + values["source_urls"] = {str(case["source_url"]) for case in schedule["scoring_cases"]} + values["queries"] = {str(case["query"]).strip().lower() for case in schedule["scoring_cases"]} + values["queries"].update(str(event["query"]).strip().lower() for event in schedule["feedback_schedule"]) + return values + + +def main() -> int: + parser = argparse.ArgumentParser(description="Audit disjoint longitudinal feedback corpus clusters.") + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--prior-fixture", type=Path, action="append", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + raise SystemExit(f"Refusing to overwrite an audit artifact: {args.output}") + manifest = _read(args.manifest) + base = args.manifest.parent + current: dict[str, dict[str, set[str]]] = {} + inputs: list[dict[str, str]] = [] + for stage in ("development", "holdout"): + for cluster in manifest[stage]["clusters"]: + fixture = base / str(cluster["fixture"]) + gold = base / str(cluster["gold"]) + key = f"{stage}:{cluster['cluster_id']}" + current[key] = _identifiers(fixture, gold) + inputs.append({"cluster": key, "fixture_sha256": _sha(fixture), "gold_sha256": _sha(gold)}) + checks: dict[str, dict[str, list[str]]] = {} + keys = sorted(current) + for index, left in enumerate(keys): + for right in keys[index + 1:]: + checks[f"{left}_vs_{right}"] = {name: sorted(current[left][name] & current[right][name]) for name in current[left]} + prior_inputs: list[dict[str, str]] = [] + for number, fixture in enumerate(args.prior_fixture, start=1): + prior = _identifiers(fixture) + prior_inputs.append({"fixture": fixture.name, "fixture_sha256": _sha(fixture)}) + for key in keys: + checks[f"{key}_vs_prior_{number}"] = {name: sorted(current[key][name] & prior[name]) for name in prior} + result: dict[str, Any] = { + "schema_version": 1, + "passed": all(not overlap for check in checks.values() for overlap in check.values()), + "inputs": {"clusters": inputs, "prior_fixtures": prior_inputs}, + "checks": checks, + "prior_usage": "fixture identifiers only; prior gold and observed result artifacts are not loaded", + } + args.output.write_text(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True) + "\n", encoding="utf-8", newline="\n") + if not result["passed"]: + raise SystemExit("Contamination audit failed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/run_longitudinal_feedback_experiment.py b/tools/run_longitudinal_feedback_experiment.py new file mode 100644 index 0000000..2d57d8f --- /dev/null +++ b/tools/run_longitudinal_feedback_experiment.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import argparse +from pathlib import Path + +from neuron_graph_rag.longitudinal_feedback_experiment import ( + run_longitudinal_feedback_development, + run_longitudinal_feedback_holdout, + write_longitudinal_feedback_result, +) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Run one registered frozen longitudinal feedback stage.") + parser.add_argument("stage", choices=("development", "holdout")) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--development-result", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + manifest = args.manifest.resolve() + expected = manifest.parent / ("d1_liplus_longitudinal_feedback_experiment.development.result.json" if args.stage == "development" else "d1_liplus_longitudinal_feedback_experiment.holdout.result.json") + if args.output.resolve() != expected.resolve(): + raise SystemExit(f"Refusing unregistered output path: {args.output}") + if args.output.exists(): + raise SystemExit(f"Refusing to overwrite an observed result: {args.output}") + if args.stage == "development": + if args.development_result is not None: + raise SystemExit("--development-result is only valid for holdout") + result = run_longitudinal_feedback_development(manifest) + else: + if args.development_result is None: + raise SystemExit("holdout requires --development-result") + result = run_longitudinal_feedback_holdout(manifest, args.development_result) + write_longitudinal_feedback_result(args.output, result) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 46f5f4f005d56e2ba7cfb9ceb219fce50d3c3bfe Mon Sep 17 00:00:00 2001 From: lipluscodex <268560960+lipluscodex@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:31:52 +0900 Subject: [PATCH 2/2] eval: record longitudinal feedback results (#37) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 固定済みdevelopmentと条件付きholdoutを各1回だけ実行した観測artifactを追加する。\nheadroom/ceilingのgate結果を保存し、default変更や一般化の主張は行わない。 --- ...eedback_experiment.development.result.json | 3776 ++++++++++++++++ ...al_feedback_experiment.holdout.result.json | 3780 +++++++++++++++++ 2 files changed, 7556 insertions(+) create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_experiment.development.result.json create mode 100644 tests/fixtures/d1_liplus_longitudinal_feedback_experiment.holdout.result.json diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.development.result.json b/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.development.result.json new file mode 100644 index 0000000..35eb01c --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.development.result.json @@ -0,0 +1,3776 @@ +{ + "ceiling_policy": { + "ceiling": "non-regression only; a pass does not evidence an additional rank improvement", + "ceiling_rank_improvement_observable": false, + "headroom": "strict final-horizon aggregate MRR improvement with per-cluster non-regression" + }, + "clusters": [ + { + "cluster_id": "lf-development-headroom-a", + "cohort": "headroom", + "control": { + "feedback": { + "credited_edges": [], + "credited_only": false, + "edge_changes": [], + "feedback_count": 3, + "no_edge_mutation": true, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-b-rival", + "lf-development-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-b-rival", + "lf-development-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-b-rival", + "lf-development-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-b-rival", + "lf-development-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "inputs": { + "fixture": "sha256:ff4cccd92c87c2f3f8caf75d10e2f9284c8e1e03ba3967a15c79794697a7c260", + "gold": "sha256:dd7eb035ecb28d72028441035812113475c695089bedea43ceeda90c047196c1", + "provenance": "sha256:92259a3c4a7953f67b4b632c02537173d19970acc13018564371d449e3d6b86a" + }, + "replay": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-b-rival", + "lf-development-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-z-target", + "lf-development-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-z-target", + "lf-development-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-z-target", + "lf-development-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "treatment": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-b-rival", + "lf-development-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-z-target", + "lf-development-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-z-target", + "lf-development-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-a-source", + "target_id": "lf-development-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-a-z-target", + "lf-development-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-a-z-target", + "id": "lf-development-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-a-source", + "id": "lf-development-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + } + }, + { + "cluster_id": "lf-development-headroom-b", + "cohort": "headroom", + "control": { + "feedback": { + "credited_edges": [], + "credited_only": false, + "edge_changes": [], + "feedback_count": 3, + "no_edge_mutation": true, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-b-rival", + "lf-development-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-b-rival", + "lf-development-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-b-rival", + "lf-development-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-b-rival", + "lf-development-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "inputs": { + "fixture": "sha256:c525765e7a54f57ea4b16ab2ff4016e64f5340fe4bbc89846081f2fbe5da4cb0", + "gold": "sha256:fc425e8fdbc566b5b7c17899793f0f018e2d87e716b6ea50ed7512d260c8ea9c", + "provenance": "sha256:7b156687006c70f3f30183c6e72f7a5e145daef83d26125669e0ab26e9051735" + }, + "replay": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-b-rival", + "lf-development-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-z-target", + "lf-development-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-z-target", + "lf-development-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-z-target", + "lf-development-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "treatment": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-b-rival", + "lf-development-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-z-target", + "lf-development-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-z-target", + "lf-development-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-headroom-b-source", + "target_id": "lf-development-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-headroom-b-z-target", + "lf-development-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-headroom-b-z-target", + "id": "lf-development-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-headroom-b-source", + "id": "lf-development-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + } + }, + { + "cluster_id": "lf-development-ceiling", + "cohort": "ceiling", + "control": { + "feedback": { + "credited_edges": [], + "credited_only": false, + "edge_changes": [], + "feedback_count": 3, + "no_edge_mutation": true, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "inputs": { + "fixture": "sha256:4436560b9a8c88d3474fcbc4401dd641a09c04aeb8f6550d8c27868bf2c7a56b", + "gold": "sha256:95faec8c3837ad05155c0e1fe0419e3f8a70a20659fa6fd0674418c5874f1527", + "provenance": "sha256:1fd5cbeb39349f29d3628ca8ff4fec4ac6968e3a6863823705330e588866ac48" + }, + "replay": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "treatment": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-development-ceiling-source", + "target_id": "lf-development-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-development-ceiling-b-target", + "lf-development-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-development-ceiling-b-target", + "id": "lf-development-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-development-ceiling-source", + "id": "lf-development-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + } + } + ], + "experiment_id": "multi-corpus-trace-credited-longitudinal-feedback-v1", + "gate": { + "ceiling_clusters_do_not_regress": true, + "ceiling_h0_control_equals_ceiling": true, + "contamination_audit_passed": true, + "credited_only_mutation_and_control_no_mutation": true, + "direct_lexical_controls_do_not_regress": true, + "directional_negative_controls_remain_empty": true, + "fixed_horizons_present": true, + "headroom_clusters_do_not_regress": true, + "headroom_final_aggregate_mrr_strictly_improves": true, + "headroom_h0_control_is_below_ceiling": true, + "projected_paths_match_at_every_horizon": true, + "same_schedule_replay_is_deterministic": true + }, + "gate_passed": true, + "holdout_status": "not_opened_candidate_selected", + "manifest_sha256": "sha256:0e47247fbe27894959b9b8ed161c9469b3febc4abb26134e63d75fff37210374", + "metrics": { + "horizon_relation_mrr": { + "0": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 0.5, + "treatment": 0.6666666666666666 + }, + "1": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 1.0, + "treatment": 1.0 + }, + "2": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 1.0, + "treatment": 1.0 + }, + "3": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 1.0, + "treatment": 1.0 + } + } + }, + "schema_version": 1, + "stage": "development" +} diff --git a/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.holdout.result.json b/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.holdout.result.json new file mode 100644 index 0000000..c613ef9 --- /dev/null +++ b/tests/fixtures/d1_liplus_longitudinal_feedback_experiment.holdout.result.json @@ -0,0 +1,3780 @@ +{ + "ceiling_policy": { + "ceiling": "non-regression only; a pass does not evidence an additional rank improvement", + "ceiling_rank_improvement_observable": false, + "headroom": "strict final-horizon aggregate MRR improvement with per-cluster non-regression" + }, + "clusters": [ + { + "cluster_id": "lf-holdout-headroom-a", + "cohort": "headroom", + "control": { + "feedback": { + "credited_edges": [], + "credited_only": false, + "edge_changes": [], + "feedback_count": 3, + "no_edge_mutation": true, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-b-rival", + "lf-holdout-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-b-rival", + "lf-holdout-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-b-rival", + "lf-holdout-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-b-rival", + "lf-holdout-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "inputs": { + "fixture": "sha256:68d3c0a516bfed8467e63243ecd091c7934222df6ce16d8cc81a5d73e50498b5", + "gold": "sha256:8874747f2e9090b674e24cdfd9fab5f4f8605562fb2f057db26aa84e03e19cc7", + "provenance": "sha256:15fedd655aa5c482d5dcef89820bf24f7912937a268e447b84e8a93c84ced724" + }, + "replay": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-b-rival", + "lf-holdout-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-z-target", + "lf-holdout-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-z-target", + "lf-holdout-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-z-target", + "lf-holdout-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "treatment": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-b-rival", + "lf-holdout-headroom-a-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-z-target", + "lf-holdout-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-z-target", + "lf-holdout-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-a-source", + "target_id": "lf-holdout-headroom-a-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-a-z-target", + "lf-holdout-headroom-a-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-a-z-target", + "id": "lf-holdout-headroom-a-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-a-source", + "id": "lf-holdout-headroom-a-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + } + }, + { + "cluster_id": "lf-holdout-headroom-b", + "cohort": "headroom", + "control": { + "feedback": { + "credited_edges": [], + "credited_only": false, + "edge_changes": [], + "feedback_count": 3, + "no_edge_mutation": true, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-b-rival", + "lf-holdout-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-b-rival", + "lf-holdout-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-b-rival", + "lf-holdout-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-b-rival", + "lf-holdout-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "inputs": { + "fixture": "sha256:ae3f7e3b63f5e01e855d573e169c1aa703ad1a2058be8fb250564937977503ea", + "gold": "sha256:bf40d4b51fb693956543e3ee2ea738a59be42405ca29ad367b9fcc7b76e3fd02", + "provenance": "sha256:2bfb8ede3cfdabf4259e6acf816a218a3eac3c0fa4d82a4e0f4ec1d94682b922" + }, + "replay": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-b-rival", + "lf-holdout-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-z-target", + "lf-holdout-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-z-target", + "lf-holdout-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-z-target", + "lf-holdout-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "treatment": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-b-rival", + "lf-holdout-headroom-b-z-target" + ], + "relation_rank": 2, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-z-target", + "lf-holdout-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-z-target", + "lf-holdout-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-headroom-b-source", + "target_id": "lf-holdout-headroom-b-z-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-headroom-b-z-target", + "lf-holdout-headroom-b-b-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-headroom-b-z-target", + "id": "lf-holdout-headroom-b-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-headroom-b-source", + "id": "lf-holdout-headroom-b-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + } + }, + { + "cluster_id": "lf-holdout-ceiling", + "cohort": "ceiling", + "control": { + "feedback": { + "credited_edges": [], + "credited_only": false, + "edge_changes": [], + "feedback_count": 3, + "no_edge_mutation": true, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "inputs": { + "fixture": "sha256:634aac3c901cdbe64f56abd89d7a66d6e7ab70b2c3daf536e26582c7341c7c07", + "gold": "sha256:f92571b85abc849a87e1c29dd67c4a5e35b032b7179567895971c530bef8efe7", + "provenance": "sha256:4cb9fa126fc9943dc2ef36647632b0b175a3fb764b6a82ca10f492b5fcb56b6b" + }, + "replay": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + }, + "treatment": { + "feedback": { + "credited_edges": [ + { + "edge_type": "supports", + "new_weight": 1.1, + "old_weight": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.204761904761905, + "old_weight": 1.1, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + }, + { + "edge_type": "supports", + "new_weight": 1.3140491617813435, + "old_weight": 1.204761904761905, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + "credited_only": true, + "edge_changes": [ + { + "after_reinforced_count": 3, + "after_weight": 1.3140491617813435, + "before_reinforced_count": 0, + "before_weight": 1.0, + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + "feedback_count": 3, + "no_edge_mutation": false, + "recorded": true + }, + "horizons": [ + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.0, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 0 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.1, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 1 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.204761904761905, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 2 + }, + { + "cases": [ + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-relation", + "lexical_rank": 3, + "observed_paths": [ + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "edge_weight": 1.3140491617813435, + "factuality": 1.0, + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "path_matched": true, + "projected_observed_paths": [ + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ], + [ + { + "edge_type": "supports", + "source_id": "lf-holdout-ceiling-source", + "target_id": "lf-holdout-ceiling-b-target" + } + ] + ], + "relation_found": true, + "relation_hits": [ + "lf-holdout-ceiling-b-target", + "lf-holdout-ceiling-z-rival" + ], + "relation_rank": 1, + "role": "relation" + }, + { + "expected_node_id": "lf-holdout-ceiling-b-target", + "id": "lf-holdout-ceiling-direct", + "lexical_rank": 1, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "direct_lexical" + }, + { + "expected_node_id": "lf-holdout-ceiling-source", + "id": "lf-holdout-ceiling-negative", + "lexical_rank": 3, + "observed_paths": [], + "path_matched": true, + "projected_observed_paths": [], + "relation_found": false, + "relation_hits": [], + "relation_rank": 3, + "role": "directional_negative" + } + ], + "horizon": 3 + } + ] + } + } + ], + "decision": { + "adopted": false, + "reason": "frozen_longitudinal_gate_passed_no_default_or_generalization_claim" + }, + "development_result_sha256": "sha256:0fee322fd99e2f55526717f0c47321dd9ac93bd793f588cea5477e6bed587404", + "experiment_id": "multi-corpus-trace-credited-longitudinal-feedback-v1", + "gate": { + "ceiling_clusters_do_not_regress": true, + "ceiling_h0_control_equals_ceiling": true, + "contamination_audit_passed": true, + "credited_only_mutation_and_control_no_mutation": true, + "direct_lexical_controls_do_not_regress": true, + "directional_negative_controls_remain_empty": true, + "fixed_horizons_present": true, + "headroom_clusters_do_not_regress": true, + "headroom_final_aggregate_mrr_strictly_improves": true, + "headroom_h0_control_is_below_ceiling": true, + "projected_paths_match_at_every_horizon": true, + "same_schedule_replay_is_deterministic": true + }, + "holdout_open_count": 1, + "manifest_sha256": "sha256:0e47247fbe27894959b9b8ed161c9469b3febc4abb26134e63d75fff37210374", + "metrics": { + "horizon_relation_mrr": { + "0": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 0.5, + "treatment": 0.6666666666666666 + }, + "1": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 1.0, + "treatment": 1.0 + }, + "2": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 1.0, + "treatment": 1.0 + }, + "3": { + "ceiling_control": 1.0, + "ceiling_treatment": 1.0, + "control": 0.6666666666666666, + "headroom_control": 0.5, + "headroom_treatment": 1.0, + "treatment": 1.0 + } + } + }, + "schema_version": 1, + "stage": "holdout" +}