Skip to content

enable mobius in fast tests and align discrepancy smoke CLI flows #2493

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
xiaoyu-work merged 9 commits into from
Jun 3, 2026
Merged

enable mobius in fast tests and align discrepancy smoke CLI flows #2493

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
4465ea4
enable mobius in fast tests
xadupre Jun 2, 2026
1272150
Add missing --test arg in mobius smoke run
Copilot Jun 2, 2026
a8d6539
Refactor shared discrepancy run invocation in smoke tests
Copilot Jun 2, 2026
bd2a22c
Use CLI --test flow for torch discrepancy smoke path
Copilot Jun 2, 2026
6b59f34
Potential fix for pull request finding
xadupre Jun 2, 2026
8aa0114
Potential fix for pull request finding
xadupre Jun 2, 2026
caf6644
Potential fix for pull request finding
xadupre Jun 2, 2026
afedbd7
Merge branch 'main' of https://github.com/microsoft/Olive into xadupr…
xadupre Jun 3, 2026
ca869b1
lint
xadupre Jun 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 150 additions & 41 deletions test/cli/test_cli_test_model_smoke.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# Licensed under the MIT License.
# --------------------------------------------------------------------------
import argparse
import importlib.util
import json
import sys
import tempfile
Expand All @@ -28,6 +29,13 @@
)
MAX_ARTIFACT_SIZE_BYTES = 1024 * 1024

# Supported exporters for the discrepancy test
EXPORTER_MODEL_BUILDER = "model_builder"
EXPORTER_MOBIUS = "mobius"
EXPORTER_TORCH = "torch_export"

_HAS_MOBIUS = importlib.util.find_spec("mobius") is not None


def _save_local_tiny_llama(model_path: Path):
model = LlamaForCausalLM(
Expand Down Expand Up @@ -133,6 +141,7 @@ def _run_documented_test_model_smoke_flow(tmp_path: Path, model_id: str):

class TestCliTestModelSmoke(unittest.TestCase):
model_ids = DEFAULT_MODEL_IDS
exporters = (EXPORTER_MODEL_BUILDER,)
Comment thread
xadupre marked this conversation as resolved.
workdir = None

def test_documented_test_model_smoke_flow(self):
Expand Down Expand Up @@ -173,7 +182,7 @@ def _assert_smoke_flows(self, tmp_path: Path):
self._assert_file_size_below_limit(run_output_dir / "model.onnx.data")

def test_model_discrepancy(self):
"""Verify that OnnxDiscrepancyCheck runs successfully when auto-injected via --test."""
"""Verify that OnnxDiscrepancyCheck runs successfully with the configured exporter."""
if self.workdir is None:
with tempfile.TemporaryDirectory() as temp_dir:
self._assert_discrepancy(Path(temp_dir))
Expand All @@ -183,48 +192,139 @@ def test_model_discrepancy(self):
self._assert_discrepancy(workdir)

def _assert_discrepancy(self, tmp_path: Path):
for model_id in self.model_ids:
with self.subTest(model_id=model_id):
model_name = model_id.replace("/", "--")
model_path = tmp_path / "models" / f"{model_name}-disc"
config_output_dir = tmp_path / f"{model_name}-disc-cfg"
test_model_dir = tmp_path / f"{model_name}-disc-test-model"
run_output_dir = tmp_path / f"{model_name}-disc-run"

_save_local_tiny_llama(model_path)
_run_cli_main(
[
"optimize",
"-m",
str(model_path),
"--device",
"cpu",
"--provider",
"CPUExecutionProvider",
"--precision",
"int4",
"--output_path",
str(config_output_dir),
"--dry_run",
]
for exporter in self.exporters:
if exporter == EXPORTER_MOBIUS and not _HAS_MOBIUS:
self.fail(
"Requested exporter 'mobius' but mobius-ai is not installed. Install mobius-ai or remove '--exporter mobius'."
)
for model_id in self.model_ids:
with self.subTest(model_id=model_id, exporter=exporter):
if exporter == EXPORTER_MODEL_BUILDER:
self._assert_discrepancy_model_builder(tmp_path, model_id)
elif exporter == EXPORTER_MOBIUS:
self._assert_discrepancy_mobius(tmp_path, model_id)
elif exporter == EXPORTER_TORCH:
self._assert_discrepancy_torch_export(tmp_path, model_id)
Comment thread
xadupre marked this conversation as resolved.
else:
self.fail(f"Unknown exporter: {exporter!r}")

config_path = config_output_dir / "config.json"
assert config_path.exists()
_set_offline_gptq_data_config(config_path)

# Run with --test; OnnxDiscrepancyCheck is auto-injected and reports discrepancy metrics (fails only if thresholds are configured)
_run_cli_main(
[
"run",
"--config",
str(config_path),
"--test",
str(test_model_dir),
"--output_path",
str(run_output_dir),
]
)
@staticmethod
def _run_discrepancy_with_test(config_path: Path, test_model_dir: Path, run_output_dir: Path):
_run_cli_main(
[
"run",
"--config",
str(config_path),
"--test",
str(test_model_dir),
"--output_path",
str(run_output_dir),
]
)

def _assert_discrepancy_model_builder(self, tmp_path: Path, model_id: str):
model_name = model_id.replace("/", "--")
model_path = tmp_path / "models" / f"{model_name}-disc"
config_output_dir = tmp_path / f"{model_name}-disc-cfg"
test_model_dir = tmp_path / f"{model_name}-disc-test-model"
run_output_dir = tmp_path / f"{model_name}-disc-run"

_save_local_tiny_llama(model_path)
_run_cli_main(
[
"optimize",
"-m",
str(model_path),
"--device",
"cpu",
"--provider",
"CPUExecutionProvider",
"--precision",
"int4",
"--output_path",
str(config_output_dir),
"--dry_run",
]
)

config_path = config_output_dir / "config.json"
assert config_path.exists()
_set_offline_gptq_data_config(config_path)

# Run with --test; OnnxDiscrepancyCheck is auto-injected and reports discrepancy metrics
self._run_discrepancy_with_test(config_path, test_model_dir, run_output_dir)

def _assert_discrepancy_mobius(self, tmp_path: Path, model_id: str):
Comment thread
xadupre marked this conversation as resolved.
model_name = model_id.replace("/", "--")
model_path = tmp_path / "models" / f"{model_name}-mobius-disc"
test_model_dir = tmp_path / f"{model_name}-mobius-disc-test-model"
run_output_dir = tmp_path / f"{model_name}-mobius-disc-run"

_save_local_tiny_llama(model_path)

run_config = {
"input_model": {
"type": "HfModel",
"model_path": str(model_path),
},
"systems": {
"local_system": {
"type": "LocalSystem",
"accelerators": [{"device": "cpu", "execution_providers": ["CPUExecutionProvider"]}],
}
},
"output_dir": str(run_output_dir),
"host": "local_system",
"target": "local_system",
"passes": {
"mobius_builder": {
"type": "MobiusBuilder",
"precision": "fp32",
"runtime": "none",
},
"discrepancy_check": {
"type": "OnnxDiscrepancyCheck",
"reference_model_path": str(test_model_dir),
},
},
}
config_path = tmp_path / f"{model_name}-mobius-disc-config.json"
config_path.write_text(json.dumps(run_config, indent=2))

self._run_discrepancy_with_test(config_path, test_model_dir, run_output_dir)

def _assert_discrepancy_torch_export(self, tmp_path: Path, model_id: str):
model_name = model_id.replace("/", "--")
model_path = tmp_path / "models" / f"{model_name}-torch-disc"
test_model_dir = tmp_path / f"{model_name}-torch-disc-test-model"
run_output_dir = tmp_path / f"{model_name}-torch-disc-run"

_save_local_tiny_llama(model_path)

run_config = {
"input_model": {
"type": "HfModel",
"model_path": str(model_path),
},
"systems": {
"local_system": {
"type": "LocalSystem",
"accelerators": [{"device": "cpu", "execution_providers": ["CPUExecutionProvider"]}],
}
},
"output_dir": str(run_output_dir),
"host": "local_system",
"target": "local_system",
"passes": {
"conversion": {
"type": "OnnxConversion",
"use_dynamo_exporter": False,
}
},
}
config_path = tmp_path / f"{model_name}-torch-disc-config.json"
config_path.write_text(json.dumps(run_config, indent=2))
self._run_discrepancy_with_test(config_path, test_model_dir, run_output_dir)

def _assert_file_size_below_limit(self, path: Path):
assert path.exists()
Expand All @@ -239,6 +339,13 @@ def _parse_args():
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("--workdir")
parser.add_argument("--model-id", dest="model_ids", action="append")
parser.add_argument(
"--exporter",
dest="exporters",
action="append",
choices=[EXPORTER_MODEL_BUILDER, EXPORTER_MOBIUS, EXPORTER_TORCH],
help="Exporter(s) to test for discrepancy check (can be repeated).",
)
return parser.parse_known_args()


Expand All @@ -248,4 +355,6 @@ def _parse_args():
TestCliTestModelSmoke.workdir = Path(parsed_args.workdir)
if parsed_args.model_ids:
TestCliTestModelSmoke.model_ids = tuple(parsed_args.model_ids)
if parsed_args.exporters:
TestCliTestModelSmoke.exporters = tuple(parsed_args.exporters)
unittest.main(argv=[__file__, *remaining])
Loading