Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 28 additions & 53 deletions kt-kernel/python/cli/commands/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -2401,7 +2401,11 @@ def fetch_hashes():
console.print()

# Step 2 & 3: Calculate local SHA256 and compare (with Progress bar)
from kt_kernel.cli.utils.model_verifier import calculate_local_sha256
from kt_kernel.cli.utils.model_verifier import (
calculate_local_sha256,
compare_local_to_official,
list_local_model_files,
)

with Progress(
SpinnerColumn(),
Expand All @@ -2414,24 +2418,20 @@ def fetch_hashes():
# Step 2: Calculate local SHA256 hashes (no timeout)
local_dir_path = Path(selected_model.path)

# Determine which files to hash
# Determine which files to hash. The remote hash set spans
# *.safetensors, *.json and *.py at any depth, keyed by repo-relative
# path, so the local scan recurses and keeps subdirectory files (#2100).
all_local_files = list_local_model_files(local_dir_path)
if files_to_verify:
# Only hash files that need re-verification
clean_filenames = {
Path(f.replace(" (missing)", "").replace(" (hash mismatch)", "").strip()).name
for f in files_to_verify
# Only hash files that need re-verification, matched by relative path.
clean_relpaths = {
f.replace(" (missing)", "").replace(" (hash mismatch)", "").strip() for f in files_to_verify
}
# Collect files matching *.safetensors, *.json, *.py
files_to_hash = []
for pattern in ["*.safetensors", "*.json", "*.py"]:
files_to_hash.extend(
[f for f in local_dir_path.glob(pattern) if f.is_file() and f.name in clean_filenames]
)
files_to_hash = [
f for f in all_local_files if f.relative_to(local_dir_path).as_posix() in clean_relpaths
]
else:
# Collect all important files: *.safetensors, *.json, *.py
files_to_hash = []
for pattern in ["*.safetensors", "*.json", "*.py"]:
files_to_hash.extend([f for f in local_dir_path.glob(pattern) if f.is_file()])
files_to_hash = all_local_files

total_files = len(files_to_hash)

Expand All @@ -2452,63 +2452,38 @@ def local_hash_callback(msg: str):

local_hashes = calculate_local_sha256(
local_dir_path,
"*.safetensors",
progress_callback=local_hash_callback,
files_list=files_to_hash if files_to_verify else None,
files_list=files_to_hash,
)

progress.remove_task(hash_task_id)
console.print(f" [green]✓ Calculated {len(local_hashes)} local file hashes[/green]")

# Step 3: Compare hashes
# If re-verifying specific files, only compare those files
# If re-verifying specific files, only compare those files (matched by
# relative path so subdirectory files are handled correctly, #2100).
if files_to_verify:
# Build set of clean filenames to verify
clean_verify_filenames = {
Path(f.replace(" (missing)", "").replace(" (hash mismatch)", "").strip()).name
for f in files_to_verify
}
# Filter official_hashes to only include files we're re-verifying
hashes_to_compare = {
filename: hash_value
for filename, hash_value in official_hashes.items()
if Path(filename).name in clean_verify_filenames
if filename in clean_relpaths
}
else:
# First-time verification: compare all files
hashes_to_compare = official_hashes

compare_task_id = progress.add_task("[blue]Comparing hashes...", total=len(hashes_to_compare))

files_failed = []
files_missing = []
files_passed = 0

for filename, official_hash in hashes_to_compare.items():
file_basename = Path(filename).name

# Find matching local file
local_hash = None
for local_file, local_hash_value in local_hashes.items():
if Path(local_file).name == file_basename:
local_hash = local_hash_value
break

if local_hash is None:
files_missing.append(filename)
if verbose:
console.print(f" [red]✗ {file_basename} (missing)[/red]")
elif local_hash.lower() != official_hash.lower():
files_failed.append(f"{filename} (hash mismatch)")
if verbose:
console.print(f" [red]✗ {file_basename} (hash mismatch)[/red]")
else:
files_passed += 1
if verbose:
console.print(f" [green]✓ {file_basename}[/green]")
files_passed, files_missing, files_mismatched = compare_local_to_official(hashes_to_compare, local_hashes)
files_failed = [f"{f} (hash mismatch)" for f in files_mismatched]

progress.update(compare_task_id, advance=1)
if verbose:
for f in files_missing:
console.print(f" [red]✗ {f} (missing)[/red]")
for f in files_mismatched:
console.print(f" [red]✗ {f} (hash mismatch)[/red]")

progress.update(compare_task_id, completed=len(hashes_to_compare))
progress.remove_task(compare_task_id)

# Build result
Expand Down
90 changes: 62 additions & 28 deletions kt-kernel/python/cli/utils/model_verifier.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,14 @@
import requests
import os
from pathlib import Path
from typing import Dict, Any, Literal, Tuple
from typing import Dict, Any, List, Literal, Tuple
from concurrent.futures import ProcessPoolExecutor, as_completed

# File suffixes whose SHA256 the remote repository publishes and we therefore
# verify locally. Must stay in sync with fetch_model_sha256, which fetches
# hashes for *.safetensors (weights), *.json (configs) and *.py (model code).
VERIFIABLE_SUFFIXES = (".safetensors", ".json", ".py")


def _compute_file_sha256(file_path: Path) -> Tuple[str, str, float]:
"""
Expand All @@ -33,6 +38,47 @@ def _compute_file_sha256(file_path: Path) -> Tuple[str, str, float]:
return file_path.name, sha256_hash.hexdigest(), file_size_mb


def list_local_model_files(local_dir: Path) -> List[Path]:
"""List every verifiable model file under local_dir, searching recursively.

The remote hash set (fetch_model_sha256) spans *.safetensors, *.json and *.py
at any depth, e.g. ``inference/config.json`` or ``encoding/tests/x.json``, so
the local scan must recurse and cover the same suffixes. A top-level,
weights-only glob reports healthy config/code files as missing (issue #2100).
"""
if not local_dir.exists():
return []
return sorted(f for f in local_dir.rglob("*") if f.is_file() and f.suffix in VERIFIABLE_SUFFIXES)


def compare_local_to_official(
official_hashes: Dict[str, str], local_hashes: Dict[str, str]
) -> Tuple[int, List[str], List[str]]:
"""Compare locally computed hashes against the official remote hashes.

Both dicts are keyed by repo-relative POSIX path (see calculate_local_sha256
and fetch_model_sha256), so each file is matched by its full relative path.
This distinguishes files that share a basename across subdirectories, e.g.
``config.json`` and ``inference/config.json`` (issue #2100); matching on the
basename alone collides them and mislabels one as missing or mismatched.

Returns (files_passed, files_missing, files_mismatched), where files_missing
and files_mismatched are lists of the offending remote relative paths.
"""
files_passed = 0
files_missing: List[str] = []
files_mismatched: List[str] = []
for filename, official_hash in official_hashes.items():
local_hash = local_hashes.get(filename)
if local_hash is None:
files_missing.append(filename)
elif local_hash.lower() != official_hash.lower():
files_mismatched.append(filename)
else:
files_passed += 1
return files_passed, files_missing, files_mismatched


def check_huggingface_connectivity(timeout: int = 5) -> Tuple[bool, str]:
"""
Check if HuggingFace is accessible.
Expand Down Expand Up @@ -268,15 +314,23 @@ def calculate_local_sha256(
# Process results as they complete
for future in as_completed(future_to_file):
completed_count += 1
file_path = future_to_file[future]
try:
filename, sha256_hash, file_size_mb = future.result()
result[filename] = sha256_hash
# Key by path relative to local_dir (POSIX) so files that share a
# basename across subdirectories (e.g. config.json vs
# inference/config.json) stay distinct and line up with the remote
# keys, which are repo-relative paths. See issue #2100.
try:
key = file_path.relative_to(local_dir).as_posix()
except ValueError:
key = filename
result[key] = sha256_hash

if progress_callback:
progress_callback(f" [{completed_count}/{total_files}] ✓ {filename} ({file_size_mb:.1f} MB)")

except Exception as e:
file_path = future_to_file[future]
if progress_callback:
progress_callback(f" [{completed_count}/{total_files}] ✗ {file_path.name} - Error: {str(e)}")

Expand Down Expand Up @@ -808,7 +862,7 @@ def fetch_with_timeout(repo_type, repo_id, use_mirror, timeout):

# Calculate local hashes and compare
local_dir = Path(user_model.path)
files_to_hash = [f for f in local_dir.glob("*.safetensors") if f.is_file()]
files_to_hash = list_local_model_files(local_dir)

with Progress(
SpinnerColumn(),
Expand All @@ -825,37 +879,17 @@ def hash_callback(msg):
if "[" in msg and "/" in msg and "]" in msg and "✓" in msg:
progress.advance(task)

local_hashes = calculate_local_sha256(local_dir, "*.safetensors", progress_callback=hash_callback)
local_hashes = calculate_local_sha256(local_dir, progress_callback=hash_callback, files_list=files_to_hash)
progress.remove_task(task)

console.print(f" [green]✓ Calculated {len(local_hashes)} local hashes[/green]")
console.print()

# Compare hashes
task = progress.add_task("[blue]Comparing hashes...", total=len(official_hashes))

files_failed = []
files_missing = []
files_passed = 0

for filename, official_hash in official_hashes.items():
file_basename = Path(filename).name
local_hash = None

for local_file, local_hash_value in local_hashes.items():
if Path(local_file).name == file_basename:
local_hash = local_hash_value
break

if local_hash is None:
files_missing.append(filename)
elif local_hash.lower() != official_hash.lower():
files_failed.append(f"{filename} (hash mismatch)")
else:
files_passed += 1

progress.advance(task)

files_passed, files_missing, files_mismatched = compare_local_to_official(official_hashes, local_hashes)
files_failed = [f"{f} (hash mismatch)" for f in files_mismatched]
progress.update(task, completed=len(official_hashes))
progress.remove_task(task)

console.print()
Expand Down
Loading