From 6018bb70e4a8f3f366a40af0445695f39cf213b5 Mon Sep 17 00:00:00 2001 From: Rashid Kaleem <230885705+arekay-nv@users.noreply.github.com> Date: Mon, 31 Aug 2026 15:39:59 -0500 Subject: [PATCH] feat(report): fingerprint events.jsonl with a content SHA-256 in result_summary.json Add `events_sha256` to the Report struct so `performance/result_summary.json` carries a SHA-256 over the bytes of the run's events.jsonl. The digest is content-only (streamed via the new `utils.hashing.sha256_file` helper), so copying or renaming the log never changes it; it is null when no event log was produced (e.g. SIGKILL before salvage). Attached in `_write_report_artifacts` after the log is salvaged to report_dir, mirroring how accuracy is attached post-hoc. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../commands/benchmark/execute.py | 9 ++- src/inference_endpoint/metrics/report.py | 6 ++ src/inference_endpoint/utils/hashing.py | 37 +++++++++++ tests/unit/metrics/test_report_builder.py | 10 +++ tests/unit/utils/test_hashing.py | 63 +++++++++++++++++++ 5 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 src/inference_endpoint/utils/hashing.py create mode 100644 tests/unit/utils/test_hashing.py diff --git a/src/inference_endpoint/commands/benchmark/execute.py b/src/inference_endpoint/commands/benchmark/execute.py index a6ad406a5..c109c9cda 100644 --- a/src/inference_endpoint/commands/benchmark/execute.py +++ b/src/inference_endpoint/commands/benchmark/execute.py @@ -106,6 +106,7 @@ SessionResult, ) from inference_endpoint.metrics.report import Report +from inference_endpoint.utils.hashing import sha256_file if TYPE_CHECKING: from inference_endpoint.async_utils.event_publisher import EventPublisherService @@ -1087,9 +1088,13 @@ def _write_report_artifacts( """Display the report and write result_summary.json + report.txt. result_summary.json is the self-complete machine-readable report (carries - qps/tps/seeds/accuracy via Report.to_json); report.txt is the full - human-readable dump; the console log shows the summary. + qps/tps/seeds/accuracy plus the events.jsonl SHA-256 via Report.to_json); + report.txt is the full human-readable dump; the console log shows the + summary. """ + report = msgspec.structs.replace( + report, events_sha256=sha256_file(ctx.report_dir / "events.jsonl") + ) report.display(fn=lambda s: logger.info(s), summary_only=True) performance_dir = ctx.report_dir / "performance" performance_dir.mkdir(parents=True, exist_ok=True) diff --git a/src/inference_endpoint/metrics/report.py b/src/inference_endpoint/metrics/report.py index 668437ee7..bd628d471 100644 --- a/src/inference_endpoint/metrics/report.py +++ b/src/inference_endpoint/metrics/report.py @@ -229,6 +229,12 @@ class Report(msgspec.Struct, frozen=True): # type: ignore[call-arg] # which can differ per audit phase — are deferred to a follow-up.) run_config: dict[str, Any] | None = None + # SHA-256 (hex) over the bytes of the run's events.jsonl, attached in + # finalize_benchmark after the log is salvaged to report_dir. Content-only, + # so copying or renaming the log does not change it. None when no event log + # was produced (e.g. SIGKILL before salvage). + events_sha256: str | None = None + # Per-dataset accuracy entries (one per scored dataset), attached after # scoring in finalize_benchmark. Accuracy is not in the metrics snapshot, so # from_snapshot leaves this empty; runs without configured scoring keep it diff --git a/src/inference_endpoint/utils/hashing.py b/src/inference_endpoint/utils/hashing.py new file mode 100644 index 000000000..e58474a7a --- /dev/null +++ b/src/inference_endpoint/utils/hashing.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Content hashing for run artifacts (e.g. fingerprinting events.jsonl).""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + + +def sha256_file(path: Path, *, chunk_size: int = 1 << 20) -> str | None: + """Hex SHA-256 over the bytes of ``path``, or None if it does not exist. + + Streamed in fixed chunks so a large file is not read into memory at once. + The digest covers file contents only — never the name or metadata — so + copying or renaming the file leaves it unchanged. + """ + if not path.exists(): + return None + digest = hashlib.sha256() + with path.open("rb") as f: + for chunk in iter(lambda: f.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() diff --git a/tests/unit/metrics/test_report_builder.py b/tests/unit/metrics/test_report_builder.py index 0102efc69..4d9341028 100644 --- a/tests/unit/metrics/test_report_builder.py +++ b/tests/unit/metrics/test_report_builder.py @@ -492,6 +492,16 @@ def test_to_json_and_display_carry_run_config(self): # Absent run_config -> null, not omitted. assert json.loads(Report.from_snapshot(snap).to_json())["run_config"] is None + def test_to_json_carries_events_sha256(self): + """result_summary.json fingerprints the run's events.jsonl. The digest is + attached post-hoc (like accuracy), so from_snapshot leaves it null and it + serializes as null until set.""" + report = _build_report(_make_registry(n_samples=5)) + assert json.loads(report.to_json())["events_sha256"] is None + + stamped = msgspec.structs.replace(report, events_sha256="deadbeef") + assert json.loads(stamped.to_json())["events_sha256"] == "deadbeef" + def test_to_json_save(self, tmp_path: Path): registry = _make_registry(n_samples=5) report = _build_report(registry) diff --git a/tests/unit/utils/test_hashing.py b/tests/unit/utils/test_hashing.py new file mode 100644 index 000000000..2ee4a0ac1 --- /dev/null +++ b/tests/unit/utils/test_hashing.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for the content-only file SHA-256 helper.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +import pytest +from inference_endpoint.utils.hashing import sha256_file + + +@pytest.mark.unit +class TestSha256File: + """Content-only SHA-256 of a file (used to fingerprint events.jsonl).""" + + def test_hashes_contents_only(self, tmp_path: Path): + """The digest is over file bytes alone, so the same content under a + different name (a copy/rename) hashes identically and matches a plain + hashlib digest of those bytes.""" + content = b'{"event": "COMPLETE"}\n' * 4 + a = tmp_path / "events.jsonl" + b = tmp_path / "events_renamed.jsonl" + a.write_bytes(content) + b.write_bytes(content) + + expected = hashlib.sha256(content).hexdigest() + assert sha256_file(a) == expected + assert sha256_file(b) == expected + + def test_differs_when_content_differs(self, tmp_path: Path): + a = tmp_path / "a.jsonl" + b = tmp_path / "b.jsonl" + a.write_bytes(b"one") + b.write_bytes(b"two") + assert sha256_file(a) != sha256_file(b) + + def test_missing_file_returns_none(self, tmp_path: Path): + """A run that never produced an event log (e.g. SIGKILL before salvage) + yields None rather than crashing the caller.""" + assert sha256_file(tmp_path / "nope.jsonl") is None + + def test_hashes_large_file_streaming(self, tmp_path: Path): + """Larger-than-one-chunk input hashes correctly (streamed read, not a + single read_bytes), matching a plain hashlib digest.""" + content = b"x" * (5 * 1024 * 1024 + 7) + path = tmp_path / "big.jsonl" + path.write_bytes(content) + assert sha256_file(path) == hashlib.sha256(content).hexdigest()