diff --git a/.github/workflows/research-repeat-v1.yml b/.github/workflows/research-repeat-v1.yml new file mode 100644 index 00000000..c1098a84 --- /dev/null +++ b/.github/workflows/research-repeat-v1.yml @@ -0,0 +1,156 @@ +name: research-repeat-v1 + +on: + workflow_dispatch: + pull_request: + paths: + - ".github/workflows/research-repeat-v1.yml" + - "research/repeat/v1/**" + - "research/data/v1/**" + - "research/corpus/v1/**" + - "scripts/research/run-repeat-v1.sh" + - "scripts/research/analyze-repeat-v1.py" + - "scripts/research/materialize-v1.sh" + - "scripts/research/verify-materialization-v1.sh" + - "scripts/research/verify-profile-lock-v1.sh" + +permissions: + contents: read + +env: + GO_VERSION: "1.25.14" + +jobs: + preflight: + name: Repeat stability preflight + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 0 + persist-credentials: false + + - name: Static checks + run: | + set -euo pipefail + bash -n scripts/research/run-repeat-v1.sh + python3 -m py_compile scripts/research/analyze-repeat-v1.py + python3 scripts/research/analyze-repeat-v1.py --self-test + jq -e . research/repeat/v1/stability-sample.json >/dev/null + + - name: Verify sample bindings + run: | + set -euo pipefail + python3 - <<'PY' + import json + from pathlib import Path + + sample=json.loads(Path("research/repeat/v1/stability-sample.json").read_text()) + plan=json.loads(Path("research/corpus/v1/study-plan.json").read_text()) + lock=json.loads(Path("research/corpus/v1/profile-identities.json").read_text()) + cases={x["id"] for x in plan["cases"]} + profiles={x["id"] for x in lock["profiles"]} + ids=set() + for row in sample["tuples"]: + assert row["id"] not in ids, f"duplicate tuple id: {row['id']}" + ids.add(row["id"]) + assert row["case_id"] in cases, row + assert row["profile_id"] in profiles, row + shard=Path("research/data/v1/processed/executions")/f"{row['case_id']}.jsonl" + matches=[json.loads(x) for x in shard.read_text().splitlines() + if x.strip() and json.loads(x)["logical_profile_id"]==row["profile_id"]] + assert len(matches)==1, row + assert sample["repeats_per_tuple"]==3 + assert len(sample["tuples"])==7 + print("[research-repeat-v1] sample bindings PASS") + PY + + execute: + name: Execute 21-attempt repeat stability sample + if: github.event_name == 'workflow_dispatch' + needs: preflight + runs-on: ubuntu-24.04 + timeout-minutes: 240 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + fetch-depth: 0 + persist-credentials: false + + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: ${{ env.GO_VERSION }} + cache: true + + - name: Install build and VM dependencies + run: | + set -euo pipefail + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential cmake clang llvm pkg-config autoconf automake libtool \ + libbpf-dev libelf-dev libzstd-dev zlib1g-dev jq curl \ + qemu-system-x86 qemu-utils cloud-image-utils openssh-client + if [[ -e /dev/kvm ]]; then + sudo chmod 0666 /dev/kvm + else + echo "::warning::/dev/kvm absent; repeat sample will use TCG." + fi + + - name: Install checksum-verified bpftool v7.5.0 + run: | + set -euo pipefail + curl -fsSL -o /tmp/bpftool-v7.5.0-amd64.tar.gz \ + https://github.com/libbpf/bpftool/releases/download/v7.5.0/bpftool-v7.5.0-amd64.tar.gz + curl -fsSL -o /tmp/bpftool-v7.5.0-amd64.tar.gz.sha256sum \ + https://github.com/libbpf/bpftool/releases/download/v7.5.0/bpftool-v7.5.0-amd64.tar.gz.sha256sum + (cd /tmp && sha256sum -c bpftool-v7.5.0-amd64.tar.gz.sha256sum) + tar -xzf /tmp/bpftool-v7.5.0-amd64.tar.gz -C /tmp + sudo install -m 0755 /tmp/bpftool /usr/local/bin/bpftool + + - name: Re-materialize frozen v1 inputs + run: | + set -euo pipefail + bash scripts/research/materialize-v1.sh dist/research-corpus-v1 + bash scripts/research/verify-materialization-v1.sh dist/research-corpus-v1 + bash scripts/research/verify-profile-lock-v1.sh + + - name: Execute repeat stability sample + run: >- + bash scripts/research/run-repeat-v1.sh + dist/research-corpus-v1 + reports/research-repeat-v1 + research/repeat/v1/stability-sample.json + + - name: Publish repeat summary + if: always() + run: | + if [[ -s reports/research-repeat-v1/normalized/stability-summary.json ]]; then + { + echo "## Research pilot v1 repeat stability" + echo + echo '~~~json' + cat reports/research-repeat-v1/normalized/stability-summary.json + echo '~~~' + } >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload repeat evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: research-repeat-v1-${{ github.run_id }} + retention-days: 90 + if-no-files-found: warn + path: | + reports/research-repeat-v1/ + dist/research-corpus-v1/materialization.json + dist/research-corpus-v1/SHA256SUMS + .bpfcompat/research-repeat-v1/**/targets/**/serial.log + .bpfcompat/research-repeat-v1/**/targets/**/qemu.log + .bpfcompat/research-repeat-v1/**/targets/**/libbpf.log + .bpfcompat/research-repeat-v1/**/targets/**/validator-result.json + .bpfcompat/research-repeat-v1/**/targets/**/validator.stderr + .bpfcompat/research-repeat-v1/**/targets/**/validator-exit-code diff --git a/research/repeat/v1/README.md b/research/repeat/v1/README.md new file mode 100644 index 00000000..9f755550 --- /dev/null +++ b/research/repeat/v1/README.md @@ -0,0 +1,46 @@ +# Pilot v1 Repeat-Run Stability + +This directory defines a **post-collection, purposefully stratified** repeat-run +sample for pilot v1. It is not a preregistered sample and it is not intended to +estimate a population-wide nondeterminism rate. + +The sample covers seven canonical execution tuples: + +- a compatible controlled baseline; +- a known ring-buffer incompatibility below the upstream boundary; +- the AlmaLinux 8 / 4.18 ring-buffer backport case; +- positive and negative Falco real-loader cases; +- a positive cilium/ebpf project-loader case; +- the Oracle logical-profile environment mismatch. + +Each tuple is repeated three times in one workflow collection, for 21 planned +attempts. Frozen v1 artifacts, loaders, validation semantics, and profile +definitions are reused. + +## Interpretation + +The repeat analyzer distinguishes two failure modes: + +1. **environment drift** — the repeated execution resolves to a different + `exact_environment_id` than the canonical run. This is environmental + change, not compatibility nondeterminism; +2. **verdict instability** — the repeated execution resolves to the same exact + environment but produces a different compatibility verdict. + +A repeat can only be called stable against the canonical observation when both +the exact environment ID and normalized verdict match. + +The workflow is manual-only because it boots real vendor VMs and is intended as +a bounded research validation, not a routine CI gate. + +## Run + +After this protocol lands on `main`: + +```bash +gh workflow run research-repeat-v1.yml --repo Kernel-Guard/bpfcompat --ref main +``` + +The workflow publishes raw repeat reports, logs, a repeat provenance record, +`repeat-executions.jsonl`, `stability-summary.json`, and generated +`RESULTS.md` as a staging Actions artifact. diff --git a/research/repeat/v1/stability-sample.json b/research/repeat/v1/stability-sample.json new file mode 100644 index 00000000..9e2a87d1 --- /dev/null +++ b/research/repeat/v1/stability-sample.json @@ -0,0 +1,60 @@ +{ + "schema_version": "bpfcompat.research.repeat-sample.v1", + "corpus_version": "v1", + "canonical_run_id": "35393833464", + "canonical_run_sha": "5de550e5cc8c659872a90fc262eb0250c229bf23", + "sampling_design": "purposeful_stratified_post_collection", + "repeats_per_tuple": 3, + "interpretation": "This is a post-collection stability check, not a preregistered or population-random sample. It intentionally covers compatible, incompatible, vendor-backport, real-loader, project-loader, and inconclusive environment-mismatch strata.", + "tuples": [ + { + "id": "positive-controlled-modern", + "case_id": "simple-pass-libbpf", + "profile_id": "ubuntu-22.04-5.15", + "stratum": "compatible_controlled", + "rationale": "Positive controlled baseline on a modern LTS kernel." + }, + { + "id": "negative-ringbuf-boundary", + "case_id": "ringbuf-modern-libbpf", + "profile_id": "ubuntu-20.04-5.4", + "stratum": "incompatible_controlled", + "rationale": "Known below-upstream-threshold ring-buffer rejection." + }, + { + "id": "positive-ringbuf-backport", + "case_id": "ringbuf-modern-libbpf", + "profile_id": "almalinux-8-4.18", + "stratum": "compatible_vendor_backport", + "rationale": "Older-numbered vendor kernel that passed the ring-buffer probe." + }, + { + "id": "negative-falco-real-loader", + "case_id": "falco-modern-bpf-scap-open", + "profile_id": "amazon-linux-2-4.14", + "stratum": "incompatible_real_loader", + "rationale": "Real Falco loader negative case." + }, + { + "id": "positive-falco-real-loader", + "case_id": "falco-modern-bpf-scap-open", + "profile_id": "ubuntu-22.04-5.15", + "stratum": "compatible_real_loader", + "rationale": "Real Falco loader positive case." + }, + { + "id": "positive-cilium-project-loader", + "case_id": "cilium-tracepoint-ebpf-go", + "profile_id": "ubuntu-22.04-5.15", + "stratum": "compatible_project_loader", + "rationale": "cilium/ebpf project-loader path." + }, + { + "id": "inconclusive-oracle-environment", + "case_id": "simple-pass-libbpf", + "profile_id": "oracle-linux-9-uek7-5.15", + "stratum": "inconclusive_environment_mismatch", + "rationale": "Canonical run requested 5.15 but booted 6.12 UEK; rerun checks whether the exact environment and inconclusive result are stable." + } + ] +} diff --git a/scripts/research/analyze-repeat-v1.py b/scripts/research/analyze-repeat-v1.py new file mode 100644 index 00000000..327d9159 --- /dev/null +++ b/scripts/research/analyze-repeat-v1.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +"""Analyze a stratified repeat-run sample against canonical pilot v1 evidence.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Any + +IMAGE_NOTE = re.compile(r"^base image sha256:\s*([0-9a-fA-F]{64})\s*$") +KERNEL_SERIES_RE = re.compile(r"^(\d+)\.(\d+)") +STATUS_TO_PRODUCER = { + "pass": "COMPATIBLE", + "fail": "INCOMPATIBLE", + "partial": "INCOMPATIBLE", + "infra_error": "INFRA_ERROR", + "unsupported": "UNSUPPORTED", +} + + +def canonical_hash(value: Any) -> str: + encoded = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode() + return "sha256:" + hashlib.sha256(encoded).hexdigest() + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + return "sha256:" + h.hexdigest() + + +def kernel_series(value: str) -> str | None: + text = value.strip() + match = KERNEL_SERIES_RE.match(text) + if not match: + return None + rest = text[match.end() :] + if rest and rest[0].isalnum(): + return None + return f"{match.group(1)}.{match.group(2)}" + + +def image_digest(target: dict[str, Any], label: str) -> str | None: + for note in target.get("notes") or []: + text = str(note).strip() + match = IMAGE_NOTE.match(text) + if match: + return "sha256:" + match.group(1).lower() + if text.lower().startswith("base image sha256:"): + raise SystemExit(f"{label}: malformed base image SHA-256 note") + return None + + +def derive_verdict(status: str, kernel_match: bool | None, env_ok: bool) -> tuple[str, str | None]: + if status == "infra_error": + return "inconclusive", "infrastructure_error" + if status == "unsupported": + return "inconclusive", "unsupported_execution_path" + if kernel_match is False: + return "inconclusive", "environment_unavailable" + if kernel_match is not True or not env_ok: + return "inconclusive", "evidence_unavailable" + if status == "pass": + return "compatible", None + if status in {"fail", "partial"}: + return "incompatible", None + raise SystemExit(f"unsupported repeat target status: {status!r}") + + +def canonical_row(data_dir: Path, case_id: str, profile_id: str) -> dict[str, Any]: + path = data_dir / "processed" / "executions" / f"{case_id}.jsonl" + matches = [ + json.loads(line) + for line in path.read_text().splitlines() + if line.strip() and json.loads(line)["logical_profile_id"] == profile_id + ] + if len(matches) != 1: + raise SystemExit( + f"canonical row lookup failed for {case_id}/{profile_id}: {len(matches)} rows" + ) + return matches[0] + + +def normalize_repeat_target( + target: dict[str, Any], + profile_revision: str, + label: str, +) -> dict[str, Any]: + profile = target.get("profile") or {} + host = target.get("host") or {} + if not isinstance(profile, dict) or not isinstance(host, dict): + raise SystemExit(f"{label}: profile/host must be objects") + + status = str(target.get("status") or "").strip().lower() + if status not in STATUS_TO_PRODUCER: + raise SystemExit(f"{label}: unsupported or missing status {status!r}") + + requested = str(profile.get("kernel_family") or "").strip() + observed = str(host.get("kernel") or "").strip() + requested_series = kernel_series(requested) + observed_series = kernel_series(observed) + kernel_match = ( + requested_series == observed_series + if requested_series is not None and observed_series is not None + else None + ) + + distro = str(profile.get("distro") or "").strip() + distro_release = str(profile.get("version") or "").strip() + arch = str(profile.get("arch") or host.get("arch") or "").strip() + image_sha = image_digest(target, label) + + env_ok = bool( + kernel_match is not None + and observed + and image_sha + and requested + and arch + and distro + and distro_release + and profile_revision + ) + + environment_id = None + if env_ok: + env_record = { + "logical_profile_id": str(target.get("profile_id") or "").strip(), + "distribution": distro, + "distribution_release": distro_release, + "architecture": arch, + "requested_kernel_family": requested, + "observed_kernel_release": observed, + "image_source": "", + "image_identity": image_sha, + "profile_revision": profile_revision, + "kernel_family_match": kernel_match, + } + environment_id = canonical_hash(env_record) + + verdict, reason = derive_verdict(status, kernel_match, env_ok) + return { + "target_status": status, + "target_verdict": STATUS_TO_PRODUCER[status], + "verdict": verdict, + "inconclusive_reason": reason, + "environment_id": environment_id, + "kernel_family_match": kernel_match, + "observed_kernel_release": observed or None, + "image_identity": image_sha, + } + + +def self_test() -> int: + assert kernel_series("5.15.0-1") == "5.15" + assert kernel_series("4.18.0-foo") == "4.18" + assert kernel_series("not-a-kernel") is None + assert derive_verdict("pass", True, True) == ("compatible", None) + assert derive_verdict("fail", True, True) == ("incompatible", None) + assert derive_verdict("pass", False, True) == ( + "inconclusive", + "environment_unavailable", + ) + a = canonical_hash({"b": 2, "a": 1}) + b = canonical_hash({"a": 1, "b": 2}) + assert a == b + print("[analyze-repeat-v1] self-test PASS") + return 0 + + +def main() -> int: + if "--self-test" in sys.argv[1:]: + return self_test() + + ap = argparse.ArgumentParser() + ap.add_argument("--reports-dir", default="reports/research-repeat-v1") + ap.add_argument("--sample", default="research/repeat/v1/stability-sample.json") + ap.add_argument("--data-dir", default="research/data/v1") + ap.add_argument("--profile-lock", default="research/corpus/v1/profile-identities.json") + ap.add_argument("--identity-lock", default="research/corpus/v1/materialized-identities.json") + ap.add_argument("--provenance") + ap.add_argument("--out-dir", default="reports/research-repeat-v1/normalized") + args = ap.parse_args() + + reports_dir = Path(args.reports_dir) + data_dir = Path(args.data_dir) + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + sample_path = Path(args.sample) + sample = json.loads(sample_path.read_text()) + if sample.get("schema_version") != "bpfcompat.research.repeat-sample.v1": + raise SystemExit("unsupported repeat sample schema") + + manifest = json.loads((data_dir / "dataset-manifest.json").read_text()) + canonical_run = manifest.get("canonical_workflow_run") or {} + if str(canonical_run.get("id")) != str(sample.get("canonical_run_id")): + raise SystemExit("repeat sample canonical run id does not match dataset manifest") + if canonical_run.get("head_sha") != sample.get("canonical_run_sha"): + raise SystemExit("repeat sample canonical run SHA does not match dataset manifest") + + provenance_path = ( + Path(args.provenance) + if args.provenance + else reports_dir / "repeat-provenance.json" + ) + if not provenance_path.is_file(): + raise SystemExit(f"missing repeat provenance: {provenance_path}") + provenance = json.loads(provenance_path.read_text()) + if provenance.get("schema_version") != "bpfcompat.research.repeat-provenance.v1": + raise SystemExit("unsupported repeat provenance schema") + if provenance.get("sample_sha256") != sha256_file(sample_path): + raise SystemExit("repeat provenance sample digest mismatch") + + identity_lock = json.loads(Path(args.identity_lock).read_text()) + identities = { + item["id"]: "sha256:" + item["sha256"].removeprefix("sha256:") + for item in identity_lock.get("artifacts") or [] + } + expected = { + "bpfcompat_cli_sha256": identities["bpfcompat-v037-cli"], + "validator_sha256": identities["bpfcompat-v037-validator"], + } + for field, digest in expected.items(): + if provenance.get(field) != digest: + raise SystemExit(f"repeat provenance identity drift: {field}") + loaders = provenance.get("loaders") or {} + if loaders.get("cilium-ebpf-v022-loader") != identities["cilium-ebpf-v022-loader"]: + raise SystemExit("repeat provenance cilium loader identity drift") + if loaders.get("falco-modern-bpf-scap-open") != identities["falco-modern-bpf-scap-open"]: + raise SystemExit("repeat provenance Falco loader identity drift") + repeats = int(sample.get("repeats_per_tuple") or 0) + tuples = sample.get("tuples") + if repeats <= 0 or not isinstance(tuples, list) or not tuples: + raise SystemExit("repeat sample must define tuples and positive repeats_per_tuple") + + lock = json.loads(Path(args.profile_lock).read_text()) + revisions = { + p["id"]: "git-blob:" + p["git_blob"] + for p in lock.get("profiles") or [] + } + + provenance_sha256 = sha256_file(provenance_path) + + rows: list[dict[str, Any]] = [] + missing: list[str] = [] + tuple_summaries: list[dict[str, Any]] = [] + + for item in tuples: + tuple_id = str(item.get("id") or "").strip() + case_id = str(item.get("case_id") or "").strip() + profile_id = str(item.get("profile_id") or "").strip() + if not tuple_id or not case_id or profile_id not in revisions: + raise SystemExit(f"invalid repeat tuple: {item!r}") + + canonical = canonical_row(data_dir, case_id, profile_id) + local_rows: list[dict[str, Any]] = [] + + for repeat in range(1, repeats + 1): + label = f"{tuple_id}/repeat-{repeat}" + report_path = reports_dir / "raw" / tuple_id / f"repeat-{repeat}.json" + if not report_path.is_file(): + missing.append(label) + continue + + report = json.loads(report_path.read_text()) + if report.get("schema_version") != "v0.1": + raise SystemExit(f"{label}: unexpected report schema") + targets = report.get("targets") + if not isinstance(targets, list) or len(targets) != 1: + raise SystemExit(f"{label}: expected exactly one target") + target = targets[0] + if str(target.get("profile_id") or "").strip() != profile_id: + raise SystemExit(f"{label}: profile mismatch") + + norm = normalize_repeat_target( + target, + revisions[profile_id], + label, + ) + if norm["environment_id"] != canonical["environment_id"]: + comparison = "environment_drift" + elif norm["verdict"] != canonical["verdict"]: + comparison = "verdict_instability" + else: + comparison = "stable_same_environment" + + row = { + "tuple_id": tuple_id, + "repeat": repeat, + "case_id": case_id, + "profile_id": profile_id, + "stratum": item.get("stratum"), + "canonical_verdict": canonical["verdict"], + "canonical_environment_id": canonical["environment_id"], + "canonical_raw_report_sha256": canonical["raw_report_sha256"], + "repeat_raw_report_sha256": sha256_file(report_path), + "repeat_provenance_sha256": provenance_sha256, + "comparison": comparison, + **norm, + } + rows.append(row) + local_rows.append(row) + + counts = Counter(r["comparison"] for r in local_rows) + tuple_summaries.append( + { + "tuple_id": tuple_id, + "case_id": case_id, + "profile_id": profile_id, + "stratum": item.get("stratum"), + "planned_repeats": repeats, + "observed_repeats": len(local_rows), + "stable_same_environment": counts["stable_same_environment"], + "environment_drift": counts["environment_drift"], + "verdict_instability": counts["verdict_instability"], + } + ) + + expected = len(tuples) * repeats + counts = Counter(r["comparison"] for r in rows) + collection_complete = len(rows) == expected and not missing + + with (out_dir / "repeat-executions.jsonl").open("w") as f: + for row in rows: + f.write(json.dumps(row, sort_keys=True) + "\n") + + summary = { + "schema_version": "bpfcompat.research.repeat-summary.v1", + "corpus_version": "v1", + "canonical_run_id": sample["canonical_run_id"], + "canonical_run_sha": sample["canonical_run_sha"], + "sampling_design": sample["sampling_design"], + "repeat_provenance_sha256": provenance_sha256, + "planned_tuples": len(tuples), + "repeats_per_tuple": repeats, + "planned_attempts": expected, + "observed_attempts": len(rows), + "collection_complete": collection_complete, + "stable_same_environment": counts["stable_same_environment"], + "environment_drift": counts["environment_drift"], + "verdict_instability": counts["verdict_instability"], + "missing": missing, + "tuple_summaries": tuple_summaries, + "interpretation": ( + "Verdict instability is counted only when the exact environment ID matches " + "the canonical run. Environment drift is reported separately and is not " + "treated as nondeterministic compatibility behavior." + ), + } + (out_dir / "stability-summary.json").write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n" + ) + + lines = [ + "# Pilot v1 repeat-run stability", + "", + "This is a purposeful post-collection stability sample, not a random population sample.", + "", + f"- Planned attempts: {expected}", + f"- Observed attempts: {len(rows)}", + f"- Stable on the same exact environment: {counts['stable_same_environment']}", + f"- Environment drift: {counts['environment_drift']}", + f"- Verdict instability on the same environment: {counts['verdict_instability']}", + "", + "Environment drift is kept separate from verdict instability.", + ] + (out_dir / "RESULTS.md").write_text("\n".join(lines) + "\n") + + print(json.dumps(summary, indent=2, sort_keys=True)) + return 0 if collection_complete else 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/research/run-repeat-v1.sh b/scripts/research/run-repeat-v1.sh new file mode 100644 index 00000000..6cee8c87 --- /dev/null +++ b/scripts/research/run-repeat-v1.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail + +BUNDLE="${1:-dist/research-corpus-v1}" +REPORTS="${2:-reports/research-repeat-v1}" +SAMPLE="${3:-research/repeat/v1/stability-sample.json}" +PLAN="research/corpus/v1/study-plan.json" + +fail() { + echo "[research-repeat-v1] $*" >&2 + exit 1 +} + +[[ -x "$BUNDLE/bin/bpfcompat-linux-amd64" ]] || fail "missing materialized BPFCompat CLI" +[[ -x "$BUNDLE/bin/bpfcompat-validator-static-linux-amd64" ]] || fail "missing materialized validator" +[[ -s "$SAMPLE" && -s "$PLAN" ]] || fail "missing repeat sample or study plan" + +bash scripts/research/verify-materialization-v1.sh "$BUNDLE" +bash scripts/research/verify-profile-lock-v1.sh + +export BPFCOMPAT_VALIDATOR_BIN="$PWD/$BUNDLE/bin/bpfcompat-validator-static-linux-amd64" +export BPFCOMPAT_VALIDATOR_SHA256="4ae1d5b838be07e6e7c304d753389a239c19eb92f6ba3bd77657e5c9583b9d04" + +BPF="$PWD/$BUNDLE/bin/bpfcompat-linux-amd64" +mkdir -p "$REPORTS/raw" "$REPORTS/logs" "$REPORTS/matrices" "$REPORTS/normalized" + +jq -n \ + --arg workflow_source_commit "$(git rev-parse HEAD)" \ + --arg sample_sha256 "sha256:$(sha256sum "$SAMPLE" | awk '{print $1}')" \ + --arg materialization_sha256 "sha256:$(sha256sum "$BUNDLE/materialization.json" | awk '{print $1}')" \ + --arg study_plan_sha256 "sha256:$(sha256sum "$PLAN" | awk '{print $1}')" \ + --arg profile_lock_sha256 "sha256:$(sha256sum research/corpus/v1/profile-identities.json | awk '{print $1}')" \ + --arg cli_sha256 "sha256:$(sha256sum "$BUNDLE/bin/bpfcompat-linux-amd64" | awk '{print $1}')" \ + --arg validator_sha256 "sha256:$(sha256sum "$BUNDLE/bin/bpfcompat-validator-static-linux-amd64" | awk '{print $1}')" \ + --arg cilium_loader_sha256 "sha256:$(sha256sum "$BUNDLE/loaders/ebpf-go-loader" | awk '{print $1}')" \ + --arg falco_loader_sha256 "sha256:$(sha256sum "$BUNDLE/loaders/scap-open" | awk '{print $1}')" \ + '{ + schema_version:"bpfcompat.research.repeat-provenance.v1", + corpus_version:"v1", + workflow_source_commit:$workflow_source_commit, + sample_sha256:$sample_sha256, + materialization_sha256:$materialization_sha256, + study_plan_sha256:$study_plan_sha256, + profile_lock_sha256:$profile_lock_sha256, + bpfcompat_cli_sha256:$cli_sha256, + validator_sha256:$validator_sha256, + loaders:{ + "cilium-ebpf-v022-loader":$cilium_loader_sha256, + "falco-modern-bpf-scap-open":$falco_loader_sha256 + } + }' > "$REPORTS/repeat-provenance.json" + +repeats="$(jq -er '.repeats_per_tuple | select(type == "number" and . > 0)' "$SAMPLE")" +tuple_rows="$(mktemp)" +trap 'rm -f "$tuple_rows"' EXIT +jq -ec '.tuples[]' "$SAMPLE" > "$tuple_rows" +[[ -s "$tuple_rows" ]] || fail "repeat sample contains no tuples" + +run_once() { + local tuple_id="$1" + local repeat="$2" + shift 2 + + local outdir="$REPORTS/raw/$tuple_id" + local logdir="$REPORTS/logs/$tuple_id" + mkdir -p "$outdir" "$logdir" + rm -f "$outdir/repeat-$repeat.json" "$logdir/repeat-$repeat.stdout.log" \ + "$logdir/repeat-$repeat.stderr.log" "$logdir/repeat-$repeat.exit-code" + + local rc=0 + if "$@" >"$logdir/repeat-$repeat.stdout.log" 2>"$logdir/repeat-$repeat.stderr.log"; then + rc=0 + else + rc=$? + fi + printf '%s\n' "$rc" > "$logdir/repeat-$repeat.exit-code" + + if [[ ! -s "$outdir/repeat-$repeat.json" ]]; then + echo "[research-repeat-v1] warning: $tuple_id repeat $repeat exited $rc without report" >&2 + else + echo "[research-repeat-v1] $tuple_id repeat $repeat captured (CLI exit $rc)" + fi +} + +while IFS= read -r tuple_json; do + tuple_id="$(jq -er '.id | select(type == "string" and length > 0)' <<<"$tuple_json")" || + fail "repeat tuple missing id" + case_id="$(jq -er '.case_id | select(type == "string" and length > 0)' <<<"$tuple_json")" || + fail "$tuple_id: missing case_id" + profile_id="$(jq -er '.profile_id | select(type == "string" and length > 0)' <<<"$tuple_json")" || + fail "$tuple_id: missing profile_id" + + [[ "$tuple_id" =~ ^[A-Za-z0-9._-]+$ ]] || fail "unsafe tuple id: $tuple_id" + [[ "$profile_id" =~ ^[A-Za-z0-9._-]+$ ]] || fail "unsafe profile id: $profile_id" + + case_json="$(jq -ec --arg id "$case_id" '.cases[] | select(.id == $id)' "$PLAN")" || + fail "$tuple_id: case not found in study plan: $case_id" + [[ -n "$case_json" ]] || fail "$tuple_id: case not found in study plan: $case_id" + + mode="$(jq -er '.mode' <<<"$case_json")" + artifact_path="$(jq -r '.artifact_path // empty' <<<"$case_json")" + manifest="$(jq -r '.manifest // empty' <<<"$case_json")" + command_binary="$(jq -r '.command_binary // empty' <<<"$case_json")" + command="$(jq -r '.command // empty' <<<"$case_json")" + + matrix="$REPORTS/matrices/$tuple_id.yaml" + cat > "$matrix" <