Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions src/inference_endpoint/commands/benchmark/execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,7 @@
SessionResult,
)
from inference_endpoint.metrics.report import Report
from inference_endpoint.utils.hashing import sha256_file

if TYPE_CHECKING:
from inference_endpoint.async_utils.event_publisher import EventPublisherService
Expand Down Expand Up @@ -1087,9 +1088,13 @@ def _write_report_artifacts(
"""Display the report and write result_summary.json + report.txt.

result_summary.json is the self-complete machine-readable report (carries
qps/tps/seeds/accuracy via Report.to_json); report.txt is the full
human-readable dump; the console log shows the summary.
qps/tps/seeds/accuracy plus the events.jsonl SHA-256 via Report.to_json);
report.txt is the full human-readable dump; the console log shows the
summary.
"""
report = msgspec.structs.replace(
report, events_sha256=sha256_file(ctx.report_dir / "events.jsonl")
)
report.display(fn=lambda s: logger.info(s), summary_only=True)
performance_dir = ctx.report_dir / "performance"
performance_dir.mkdir(parents=True, exist_ok=True)
Expand Down
6 changes: 6 additions & 0 deletions src/inference_endpoint/metrics/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,12 @@ class Report(msgspec.Struct, frozen=True): # type: ignore[call-arg]
# which can differ per audit phase — are deferred to a follow-up.)
run_config: dict[str, Any] | None = None

# SHA-256 (hex) over the bytes of the run's events.jsonl, attached in
# finalize_benchmark after the log is salvaged to report_dir. Content-only,
# so copying or renaming the log does not change it. None when no event log
# was produced (e.g. SIGKILL before salvage).
events_sha256: str | None = None

# Per-dataset accuracy entries (one per scored dataset), attached after
# scoring in finalize_benchmark. Accuracy is not in the metrics snapshot, so
# from_snapshot leaves this empty; runs without configured scoring keep it
Expand Down
37 changes: 37 additions & 0 deletions src/inference_endpoint/utils/hashing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Content hashing for run artifacts (e.g. fingerprinting events.jsonl)."""

from __future__ import annotations

import hashlib
from pathlib import Path


def sha256_file(path: Path, *, chunk_size: int = 1 << 20) -> str | None:
"""Hex SHA-256 over the bytes of ``path``, or None if it does not exist.

Streamed in fixed chunks so a large file is not read into memory at once.
The digest covers file contents only — never the name or metadata — so
copying or renaming the file leaves it unchanged.
"""
if not path.exists():
return None
digest = hashlib.sha256()
with path.open("rb") as f:
for chunk in iter(lambda: f.read(chunk_size), b""):
digest.update(chunk)
return digest.hexdigest()
10 changes: 10 additions & 0 deletions tests/unit/metrics/test_report_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,16 @@ def test_to_json_and_display_carry_run_config(self):
# Absent run_config -> null, not omitted.
assert json.loads(Report.from_snapshot(snap).to_json())["run_config"] is None

def test_to_json_carries_events_sha256(self):
"""result_summary.json fingerprints the run's events.jsonl. The digest is
attached post-hoc (like accuracy), so from_snapshot leaves it null and it
serializes as null until set."""
report = _build_report(_make_registry(n_samples=5))
assert json.loads(report.to_json())["events_sha256"] is None

stamped = msgspec.structs.replace(report, events_sha256="deadbeef")
assert json.loads(stamped.to_json())["events_sha256"] == "deadbeef"

def test_to_json_save(self, tmp_path: Path):
registry = _make_registry(n_samples=5)
report = _build_report(registry)
Expand Down
63 changes: 63 additions & 0 deletions tests/unit/utils/test_hashing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for the content-only file SHA-256 helper."""

from __future__ import annotations

import hashlib
from pathlib import Path

import pytest
from inference_endpoint.utils.hashing import sha256_file


@pytest.mark.unit
class TestSha256File:
"""Content-only SHA-256 of a file (used to fingerprint events.jsonl)."""

def test_hashes_contents_only(self, tmp_path: Path):
"""The digest is over file bytes alone, so the same content under a
different name (a copy/rename) hashes identically and matches a plain
hashlib digest of those bytes."""
content = b'{"event": "COMPLETE"}\n' * 4
a = tmp_path / "events.jsonl"
b = tmp_path / "events_renamed.jsonl"
a.write_bytes(content)
b.write_bytes(content)

expected = hashlib.sha256(content).hexdigest()
assert sha256_file(a) == expected
assert sha256_file(b) == expected

def test_differs_when_content_differs(self, tmp_path: Path):
a = tmp_path / "a.jsonl"
b = tmp_path / "b.jsonl"
a.write_bytes(b"one")
b.write_bytes(b"two")
assert sha256_file(a) != sha256_file(b)

def test_missing_file_returns_none(self, tmp_path: Path):
"""A run that never produced an event log (e.g. SIGKILL before salvage)
yields None rather than crashing the caller."""
assert sha256_file(tmp_path / "nope.jsonl") is None

def test_hashes_large_file_streaming(self, tmp_path: Path):
"""Larger-than-one-chunk input hashes correctly (streamed read, not a
single read_bytes), matching a plain hashlib digest."""
content = b"x" * (5 * 1024 * 1024 + 7)
path = tmp_path / "big.jsonl"
path.write_bytes(content)
assert sha256_file(path) == hashlib.sha256(content).hexdigest()
Loading