Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
71c2c91
update SSH testing: improve CI environment compatibility with mock fa…
allnes Aug 17, 2025
2b96e3d
update SSH testing: improve CI environment compatibility (#18)
allnes Aug 17, 2025
28a0995
refactor tests and core utilities: use `datetime.now(timezone.utc)` f…
allnes Aug 17, 2025
9aae20a
Merge branch 'main' into an/windows-support
allnes Aug 17, 2025
86e3580
refactor tests and core utilities: remove unnecessary whitespace, sim…
allnes Aug 17, 2025
e74f1bb
update SSH setup and testing: add Windows support with PowerShell scr…
allnes Aug 17, 2025
2b05fba
refactor tests: remove unnecessary whitespace in SSH-related test fil…
allnes Aug 17, 2025
7adbf54
refactor SSH setup: simplify script for CI environments by removing O…
allnes Aug 17, 2025
a154030
enhance CLI and CI encoding handling: add UTF-8 defaults for Windows,…
allnes Aug 17, 2025
635b881
remove unnecessary whitespace: clean up extra blank lines and trailin…
allnes Aug 17, 2025
65c6c19
handle exceptions explicitly: replace bare `except` with `except Exce…
allnes Aug 17, 2025
67d0618
refactor SSH tests and setup: simplify platform checks, enhance loggi…
allnes Aug 17, 2025
ffc0f08
refactor SSH tests and setup: simplify platform checks, enhance loggi…
allnes Aug 17, 2025
0615ef9
remove redundant platform checks: simplify `test_ssh_in_ci` by removi…
allnes Aug 17, 2025
ad16ffc
update SSH testing: remove unused variables in test scripts for clean…
allnes Aug 17, 2025
6f1fbc6
add shell specification for SSH commands in CI workflow for consisten…
allnes Aug 17, 2025
610648c
install and configure OpenSSH Server: enhance CI SSH setup with serve…
allnes Aug 17, 2025
69e7895
clean up `test_ssh_in_ci`: adjust formatting for consistent style and…
allnes Aug 17, 2025
73fa7a1
update SSH configurations and tests: replace `localhost` with `127.0.…
allnes Aug 17, 2025
c8502c0
validate SSH device config: ensure `host` is specified in device conf…
allnes Aug 17, 2025
c2215e8
format SSH code and tests: adjust indentation, replace single quotes …
allnes Aug 17, 2025
0030500
add platform-specific logic to SSH tests: update `test_main_setup` an…
allnes Aug 17, 2025
df89077
refactor SSH tests: adjust formatting in `test_generate_ssh_config` f…
allnes Aug 17, 2025
0e56d51
remove SSH localhost experiments and enhance mock mode: delete unused…
allnes Aug 17, 2025
bfdcecf
remove generate_ssh_config.py and associated files: delete unused scr…
allnes Aug 17, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions .github/codecov.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ ignore:
- "**/__pycache__"
- "**/*.pyc"
- "setup.py"
- "scripts/generate_ssh_config.py"
- "scripts/test_ssh_device_ci.py"

coverage:
status:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/bench.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ jobs:
ci-matrix:
strategy:
matrix:
os: [ubuntu-latest, macos-latest] # , windows-latest]
os: [ubuntu-latest, macos-latest, windows-latest]
uses: ./.github/workflows/ci-orchestrator.yml
with:
os: ${{ matrix.os }}
Expand Down
24 changes: 10 additions & 14 deletions .github/workflows/stage-device-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,24 +28,20 @@ jobs:
pip install -r requirements.txt
pip install -e .

- name: Set up SSH server
- name: Test SSH with Paramiko test server
shell: bash
run: |
python scripts/generate_ssh_config.py --type setup
bash scripts/setup_ssh_ci.sh || echo "SSH setup had warnings, continuing..."

- name: List SSH devices
run: |
ovmobilebench list-ssh-devices || echo "Command not yet implemented"

- name: Test SSH deployment
run: |
python scripts/generate_ssh_config.py --type test
python scripts/test_ssh_device_ci.py
set -e
python tests/test_ssh_device_ci.py

- name: Run benchmark dry-run via SSH
shell: bash
env:
PYTHONIOENCODING: utf-8
PYTHONUTF8: 1
run: |
python scripts/generate_ssh_config.py --type config
ovmobilebench all -c experiments/ssh_localhost_ci.yaml --dry-run || true
set -e
ovmobilebench all -c experiments/ssh_test_ci.yaml --dry-run

- name: Upload SSH test results
if: always()
Expand Down
4 changes: 1 addition & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,5 @@ dmypy.json
.claude
CLAUDE.md

# Generated CI configs
experiments/ssh_localhost_ci.yaml
# Test results
experiments/results/
scripts/setup_ssh_ci.sh
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ cat experiments/results/*.csv
| Windows | x86_64 | Android | ARM64 | ADB | adbutils | ✅ Stable |
| Linux | x86_64 | Linux | ARM64/ARM32 | SSH | paramiko | ✅ Stable |
| macOS | x86_64/ARM64 | Linux | ARM64/ARM32 | SSH | paramiko | ✅ Stable |
| Windows | x86_64 | Linux | ARM64/ARM32 | SSH | paramiko | ✅ Stable |
| Any | Any | iOS | ARM64 | USB | - | 🚧 Planned |

## 📋 Requirements
Expand Down
2 changes: 1 addition & 1 deletion docs/ci-cd.md
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,7 @@ def collect_metrics(results_path, metadata):
def send_to_influxdb(metrics):
from influxdb import InfluxDBClient

client = InfluxDBClient('localhost', 8086, database='ovmobilebench')
client = InfluxDBClient('127.0.0.1', 8086, database='ovmobilebench')

points = []
for result in metrics['results']:
Expand Down
44 changes: 0 additions & 44 deletions experiments/ssh_localhost.yaml

This file was deleted.

43 changes: 43 additions & 0 deletions experiments/ssh_test_ci.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# SSH test configuration for CI
project:
name: ssh-test-ci
run_id: ci-test

# SSH device configuration for mock testing
device:
type: linux_ssh
host: 127.0.0.1
username: testuser
push_dir: /tmp/ovmobilebench

# Build configuration (disabled for CI)
build:
enabled: false
openvino_repo: /tmp/openvino

# Dummy models for testing
models:
- name: dummy_model
path: /tmp/dummy_model.xml
precision: FP32

# Run configuration
run:
repeats: 1
warmup: 0
cooldown_sec: 0
matrix:
niter: [10]
nstreams: ["1"]
device: ["CPU"]

# Reporting
report:
sinks:
- type: csv
path: experiments/results/ssh_test.csv
- type: json
path: experiments/results/ssh_test.json
tags:
test_type: ssh_test_ci
ci: true
82 changes: 61 additions & 21 deletions ovmobilebench/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
# Apply typer compatibility patch
from ovmobilebench import typer_patch # noqa: F401

import os
import sys
import typer
from pathlib import Path
from typing import Optional
Expand All @@ -12,14 +14,27 @@
from ovmobilebench.config.loader import load_experiment
from ovmobilebench.pipeline import Pipeline

# Set UTF-8 encoding for Windows
if sys.platform == "win32":
os.environ["PYTHONIOENCODING"] = "utf-8"
# Also set console code page to UTF-8 if possible
try:
import subprocess

subprocess.run("chcp 65001", shell=True, capture_output=True)
except Exception:
pass

app = typer.Typer(
name="ovmobilebench",
help="End-to-end benchmarking pipeline for OpenVINO on mobile devices",
add_completion=False,
pretty_exceptions_enable=False, # Disable pretty exceptions
rich_markup_mode=None, # Disable Rich formatting
)
console = Console()

# Configure console with safe encoding for Windows
console = Console(legacy_windows=True if sys.platform == "win32" else None)


@app.command()
Expand All @@ -33,7 +48,7 @@ def build(
cfg = load_experiment(config)
pipeline = Pipeline(cfg, verbose=verbose, dry_run=dry_run)
pipeline.build()
console.print("[bold green] Build completed[/bold green]")
console.print("[bold green][OK] Build completed[/bold green]")


@app.command()
Expand All @@ -47,7 +62,7 @@ def package(
cfg = load_experiment(config)
pipeline = Pipeline(cfg, verbose=verbose, dry_run=dry_run)
pipeline.package()
console.print("[bold green] Package created[/bold green]")
console.print("[bold green][OK] Package created[/bold green]")


@app.command()
Expand All @@ -61,7 +76,7 @@ def deploy(
cfg = load_experiment(config)
pipeline = Pipeline(cfg, verbose=verbose, dry_run=dry_run)
pipeline.deploy()
console.print("[bold green] Deployment completed[/bold green]")
console.print("[bold green][OK] Deployment completed[/bold green]")


@app.command()
Expand All @@ -77,7 +92,7 @@ def run(
cfg = load_experiment(config)
pipeline = Pipeline(cfg, verbose=verbose, dry_run=dry_run)
pipeline.run(timeout=timeout, cooldown=cooldown)
console.print("[bold green] Benchmarks completed[/bold green]")
console.print("[bold green][OK] Benchmarks completed[/bold green]")


@app.command()
Expand All @@ -90,7 +105,7 @@ def report(
cfg = load_experiment(config)
pipeline = Pipeline(cfg, verbose=verbose)
pipeline.report()
console.print("[bold green] Reports generated[/bold green]")
console.print("[bold green][OK] Reports generated[/bold green]")


@app.command()
Expand All @@ -102,11 +117,10 @@ def all(
cooldown: Optional[int] = typer.Option(None, "--cooldown", help="Cooldown between runs"),
):
"""Execute complete pipeline: build, package, deploy, run, and report."""
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
console=console,
) as progress:
# Check if we're in CI environment
is_ci = os.environ.get("CI", "").lower() == "true"

try:
cfg = load_experiment(config)
pipeline = Pipeline(cfg, verbose=verbose, dry_run=dry_run)

Expand All @@ -118,16 +132,42 @@ def all(
("Generating reports...", pipeline.report),
]

for description, stage_func in stages:
task = progress.add_task(description, total=None)
try:
stage_func()
progress.update(task, completed=True)
except Exception as e:
console.print(f"[bold red]✗ {description} failed: {e}[/bold red]")
raise

console.print("[bold green]✓ Pipeline completed successfully[/bold green]")
if is_ci or verbose:
# Simple output for CI or verbose mode
for description, stage_func in stages:
print(f"[*] {description}")
try:
stage_func()
print(f"[OK] {description} completed")
except Exception as e:
print(f"[FAIL] {description} failed: {e}")
raise
print("[OK] Pipeline completed successfully")
else:
# Rich progress bar for interactive use
spinner = SpinnerColumn(spinner_name="dots" if sys.platform == "win32" else "aesthetic")

with Progress(
spinner,
TextColumn("[progress.description]{task.description}"),
console=console,
transient=True, # Clear progress when done
) as progress:
for description, stage_func in stages:
task = progress.add_task(description, total=None)
try:
stage_func()
progress.update(task, completed=True)
except Exception as e:
console.print(f"[bold red][FAIL] {description} failed: {e}[/bold red]")
raise

console.print("[bold green][OK] Pipeline completed successfully[/bold green]")
except UnicodeEncodeError as e:
# Fallback for encoding errors
print(f"Encoding error: {e}")
print("Pipeline failed due to encoding issues. Try setting PYTHONIOENCODING=utf-8")
sys.exit(1)


@app.command()
Expand Down
2 changes: 1 addition & 1 deletion ovmobilebench/config/loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
def load_yaml(path: Path) -> Dict[str, Any]:
"""Load YAML configuration file."""
if not path.exists():
raise FileNotFoundError(f"Configuration file not found: {path}")
raise FileNotFoundError(f"Configuration file not found: {path.as_posix()}")

with open(path, "r") as f:
data: Dict[str, Any] = yaml.safe_load(f)
Expand Down
8 changes: 4 additions & 4 deletions ovmobilebench/core/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
import hashlib
from pathlib import Path
from typing import Dict, Any, Optional, List
from datetime import datetime
from datetime import datetime, timezone

from ovmobilebench.core.fs import ensure_dir, atomic_write

Expand Down Expand Up @@ -120,9 +120,9 @@ def register_artifact(
# Prepare artifact record
record: Dict[str, Any] = {
"type": artifact_type,
"path": str(path.relative_to(self.base_dir)),
"path": path.relative_to(self.base_dir).as_posix(),
"size": path.stat().st_size if path.is_file() else None,
"created_at": datetime.utcnow().isoformat(),
"created_at": datetime.now(timezone.utc).isoformat(),
"checksum": artifact_id,
}

Expand Down Expand Up @@ -192,7 +192,7 @@ def cleanup_old_artifacts(self, days: int = 30) -> int:
"""
from datetime import timedelta

cutoff = datetime.utcnow() - timedelta(days=days)
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
artifacts = self.load_metadata().get("artifacts", {})
to_remove = []

Expand Down
4 changes: 2 additions & 2 deletions ovmobilebench/core/logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,15 @@
import json
from pathlib import Path
from typing import Optional, List
from datetime import datetime
from datetime import datetime, timezone


class JSONFormatter(logging.Formatter):
"""JSON log formatter."""

def format(self, record):
log_obj = {
"timestamp": datetime.utcnow().isoformat(),
"timestamp": datetime.now(timezone.utc).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
Expand Down
Loading
Loading