diff --git a/.github/actions/setup-python-cached/action.yml b/.github/actions/setup-python-cached/action.yml new file mode 100644 index 0000000..cebaf6b --- /dev/null +++ b/.github/actions/setup-python-cached/action.yml @@ -0,0 +1,98 @@ +name: 'Setup Python with Cached Dependencies' +description: 'Setup Python and cache entire site-packages directory' + +inputs: + python-version: + description: 'Python version to use' + required: false + default: '3.11' + +runs: + using: 'composite' + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + - name: Get Python paths + id: python-paths + shell: bash + run: | + # Get Python paths using Python + python << 'EOF' + import os + import sys + import site + import subprocess + + # Get site-packages path + site_packages = site.getsitepackages()[0] + + # Get pip cache dir + pip_cache = subprocess.check_output([sys.executable, "-m", "pip", "cache", "dir"]).decode().strip() + + # Output for GitHub Actions + output_file = os.environ.get('GITHUB_OUTPUT', '') + if output_file: + with open(output_file, 'a') as f: + f.write(f"site-packages={site_packages}\n") + f.write(f"pip-cache={pip_cache}\n") + f.write(f"python-version={sys.version}\n") + EOF + + - name: Cache Python environment + uses: actions/cache@v3 + id: python-cache + with: + path: | + ${{ steps.python-paths.outputs.site-packages }} + ${{ steps.python-paths.outputs.pip-cache }} + ~/.local/bin + ~/.local/lib + ~/Library/Python + /opt/hostedtoolcache/Python/*/x64/bin + /opt/hostedtoolcache/Python/*/x64/lib/python*/site-packages + key: ${{ runner.os }}-py${{ inputs.python-version }}-${{ hashFiles('requirements.txt', 'pyproject.toml') }}-v2 + restore-keys: | + ${{ runner.os }}-py${{ inputs.python-version }}- + + - name: Install dependencies + shell: bash + run: | + echo "Cache hit: ${{ steps.python-cache.outputs.cache-hit }}" + echo "Python version: ${{ steps.python-paths.outputs.python-version }}" + echo "Site packages: ${{ steps.python-paths.outputs.site-packages }}" + + # Always ensure pip is up to date + pip install --upgrade pip + + if [[ "${{ steps.python-cache.outputs.cache-hit }}" != "true" ]]; then + echo "Cache miss - installing all dependencies" + pip install -r requirements.txt + pip install -e . + else + echo "Cache hit - verifying and reinstalling if needed" + # Check if black is available + if ! command -v black &> /dev/null; then + echo "Tools not in PATH, reinstalling..." + pip install -r requirements.txt + fi + # Always reinstall the local package in case code changed + pip install -e . --no-deps --force-reinstall + fi + + # Ensure tools are in PATH + export PATH="$HOME/.local/bin:$PATH" + echo "PATH=$HOME/.local/bin:$PATH" >> $GITHUB_ENV + + - name: Verify installation + shell: bash + run: | + echo "=== Installed packages ===" + pip list 2>/dev/null | head -20 || true + echo "..." + echo "=== Package location ===" + python -c "import ovmobilebench; print(f'Package installed at: {ovmobilebench.__file__}')" || echo "Package not found" + echo "=== CLI availability ===" + which ovmobilebench || echo "CLI not in PATH" \ No newline at end of file diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6a1cdc6..8f5b0e0 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -16,7 +16,8 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] # , windows-latest] - uses: ./.github/workflows/reusable-ci.yml + uses: ./.github/workflows/ci-orchestrator.yml with: os: ${{ matrix.os }} device_serial: ${{ github.event.inputs.device_serial || 'emulator-5554' }} + secrets: inherit diff --git a/.github/workflows/ci-orchestrator.yml b/.github/workflows/ci-orchestrator.yml new file mode 100644 index 0000000..6c29a44 --- /dev/null +++ b/.github/workflows/ci-orchestrator.yml @@ -0,0 +1,45 @@ +name: CI Orchestrator + +on: + workflow_call: + inputs: + os: + required: true + type: string + device_serial: + required: false + type: string + default: 'emulator-5554' + +jobs: + # Stage 1: Lint and Test + lint-test: + uses: ./.github/workflows/stage-lint-test.yml + with: + os: ${{ inputs.os }} + secrets: inherit + + # Stage 2: Build Package (runs in parallel with validation) + build: + needs: lint-test + uses: ./.github/workflows/stage-build.yml + with: + os: ${{ inputs.os }} + secrets: inherit + + # Stage 3: Validation (runs in parallel with build) + validation: + needs: lint-test + uses: ./.github/workflows/stage-validation.yml + with: + os: ${{ inputs.os }} + secrets: inherit + + # Stage 4: Device Tests (runs after build and validation) + device-tests: + needs: [build, validation] + uses: ./.github/workflows/stage-device-tests.yml + with: + os: ${{ inputs.os }} + device_serial: ${{ inputs.device_serial }} + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/reusable-ci.yml b/.github/workflows/reusable-ci.yml deleted file mode 100644 index 35526f8..0000000 --- a/.github/workflows/reusable-ci.yml +++ /dev/null @@ -1,204 +0,0 @@ -name: Reusable CI Workflow - -on: - workflow_call: - inputs: - os: - required: true - type: string - device_serial: - required: false - type: string - -jobs: - lint-and-test: - runs-on: ${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Cache pip - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('pyproject.toml') }} - restore-keys: | - ${{ runner.os }}-pip- - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r requirements.txt - pip install -e . - - - name: Run Black - run: black --check ovmobilebench tests - - - name: Run Ruff - run: ruff check ovmobilebench tests - - - name: Run MyPy - run: mypy ovmobilebench --ignore-missing-imports - - - name: Run tests - run: pytest tests/ -v --cov=ovmobilebench --cov=scripts --cov-report=xml --cov-report=term-missing - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v4 - with: - file: ./coverage.xml - flags: unittests - name: codecov-${{ inputs.os }} - fail_ci_if_error: false - token: ${{ secrets.CODECOV_TOKEN }} - verbose: true - - build-package: - needs: lint-and-test - runs-on: ${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install build dependencies - run: | - pip install --upgrade pip - pip install build setuptools wheel - - - name: Build package - run: python -m build - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: dist-${{ inputs.os }} - path: dist/ - retention-days: 7 - - dry-run: - needs: lint-and-test - runs-on: ${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r requirements.txt - pip install -e . - - - name: Validate example config - run: | - python -c "from ovmobilebench.config.loader import load_experiment; load_experiment('experiments/android_example.yaml')" - - - name: CLI help test - run: | - ovmobilebench --help - ovmobilebench build --help - ovmobilebench run --help - - device-test-ssh: - if: inputs.os == 'ubuntu-latest' # SSH tests only on Ubuntu for now - needs: - - build-package - - dry-run - runs-on: ${{ inputs.os }} - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r requirements.txt - pip install -e . - - - name: Set up SSH server - run: | - # Generate and run SSH setup script - python scripts/generate_ssh_config.py --type setup - bash scripts/setup_ssh_ci.sh || echo "SSH setup had warnings, continuing..." - - - name: List SSH devices - run: | - ovmobilebench list-ssh-devices || echo "Command not yet implemented" - - - name: Test SSH deployment - run: | - # Generate and run SSH test script - python scripts/generate_ssh_config.py --type test - python scripts/test_ssh_device_ci.py - - - name: Run benchmark dry-run via SSH - run: | - # Generate SSH config using Python script - python scripts/generate_ssh_config.py --type config - - # Run in dry-run mode - ovmobilebench all -c experiments/ssh_localhost_ci.yaml --dry-run || true - - - name: Upload SSH test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: benchmark-results-ssh-${{ inputs.os }} - path: experiments/results/ - retention-days: 30 - - # Optional: Run on a self-hosted runner with a real device - device-test-adb: - # Only run if explicitly triggered and only once (not for each OS in matrix) - if: (github.event_name == 'workflow_dispatch' || contains(github.event.head_commit.message, '[device-test-adb]')) && inputs.os == 'ubuntu-latest' - needs: - - build-package - - dry-run - runs-on: self-hosted - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - pip install --upgrade pip - pip install -r requirements.txt - pip install -e . - - - name: Check ADB devices - run: adb devices - - - name: Run minimal benchmark - env: - DEVICE_SERIAL: ${{ inputs.device_serial }} - run: | - ovmobilebench list-devices - # Uncomment when ready: - # ovmobilebench all -c experiments/android_example.yaml --dry-run - - - name: Upload results - if: always() - uses: actions/upload-artifact@v4 - with: - name: benchmark-results-adb-${{ inputs.os }} - path: experiments/results/ - retention-days: 30 diff --git a/.github/workflows/stage-build.yml b/.github/workflows/stage-build.yml new file mode 100644 index 0000000..5a73373 --- /dev/null +++ b/.github/workflows/stage-build.yml @@ -0,0 +1,27 @@ +name: Build Package + +on: + workflow_call: + inputs: + os: + required: true + type: string + +jobs: + build-package: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + + - name: Setup Python with cached dependencies + uses: ./.github/actions/setup-python-cached + + - name: Build package + run: python -m build + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-${{ inputs.os }} + path: dist/ + retention-days: 7 \ No newline at end of file diff --git a/.github/workflows/stage-device-tests.yml b/.github/workflows/stage-device-tests.yml new file mode 100644 index 0000000..48cc4b0 --- /dev/null +++ b/.github/workflows/stage-device-tests.yml @@ -0,0 +1,80 @@ +name: Device Tests + +on: + workflow_call: + inputs: + os: + required: true + type: string + device_serial: + required: false + type: string + default: 'emulator-5554' + +jobs: + device-test-ssh: + if: inputs.os == 'ubuntu-latest' + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + + - name: Setup Python with cached dependencies + uses: ./.github/actions/setup-python-cached + + - name: Set up SSH server + run: | + python scripts/generate_ssh_config.py --type setup + bash scripts/setup_ssh_ci.sh || echo "SSH setup had warnings, continuing..." + + - name: List SSH devices + run: | + ovmobilebench list-ssh-devices || echo "Command not yet implemented" + + - name: Test SSH deployment + run: | + python scripts/generate_ssh_config.py --type test + python scripts/test_ssh_device_ci.py + + - name: Run benchmark dry-run via SSH + run: | + python scripts/generate_ssh_config.py --type config + ovmobilebench all -c experiments/ssh_localhost_ci.yaml --dry-run || true + + - name: Upload SSH test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-ssh-${{ inputs.os }} + path: experiments/results/ + retention-days: 30 + + device-test-adb: + if: | + (github.event_name == 'workflow_dispatch' || + contains(github.event.head_commit.message, '[device-test-adb]')) && + inputs.os == 'ubuntu-latest' + runs-on: self-hosted + steps: + - uses: actions/checkout@v4 + + - name: Setup Python with cached dependencies + uses: ./.github/actions/setup-python-cached + + - name: Check ADB devices + run: adb devices + + - name: Run minimal benchmark + env: + DEVICE_SERIAL: ${{ inputs.device_serial }} + run: | + ovmobilebench list-devices + # Uncomment when ready: + # ovmobilebench all -c experiments/android_example.yaml --dry-run + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results-adb-${{ inputs.os }} + path: experiments/results/ + retention-days: 30 \ No newline at end of file diff --git a/.github/workflows/stage-lint-test.yml b/.github/workflows/stage-lint-test.yml new file mode 100644 index 0000000..0087ea4 --- /dev/null +++ b/.github/workflows/stage-lint-test.yml @@ -0,0 +1,41 @@ +name: Lint and Test + +on: + workflow_call: + inputs: + os: + required: true + type: string + +jobs: + lint-and-test: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + + - name: Setup Python with cached dependencies + uses: ./.github/actions/setup-python-cached + + - name: Run Black + run: black --check ovmobilebench tests + + - name: Run Ruff + run: ruff check ovmobilebench tests + + - name: Run MyPy + run: mypy ovmobilebench --ignore-missing-imports + + - name: Run tests + run: pytest tests/ -v --cov=ovmobilebench --cov=scripts --cov-report=xml --cov-report=term-missing + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + files: ./coverage.xml + flags: unittests + name: codecov-${{ inputs.os }} + fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/stage-validation.yml b/.github/workflows/stage-validation.yml new file mode 100644 index 0000000..e3a94df --- /dev/null +++ b/.github/workflows/stage-validation.yml @@ -0,0 +1,35 @@ +name: Validation + +on: + workflow_call: + inputs: + os: + required: true + type: string + +jobs: + dry-run: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + + - name: Setup Python with cached dependencies + uses: ./.github/actions/setup-python-cached + + - name: Validate example config + run: | + python -c "from ovmobilebench.config.loader import load_experiment; load_experiment('experiments/android_example.yaml')" + + - name: CLI help test + run: | + ovmobilebench --help + ovmobilebench build --help + ovmobilebench run --help + + - name: Check CLI commands + run: | + ovmobilebench list-devices --help + ovmobilebench all --help + ovmobilebench package --help + ovmobilebench deploy --help + ovmobilebench report --help \ No newline at end of file diff --git a/.codecov.yml b/codecov.yml similarity index 87% rename from .codecov.yml rename to codecov.yml index 2f46163..620010b 100644 --- a/.codecov.yml +++ b/codecov.yml @@ -1,22 +1,23 @@ codecov: require_ci_to_pass: yes + token: required coverage: precision: 2 round: down - range: "99...100" + range: "70...100" status: project: default: - target: 99% + target: auto threshold: 1% paths: - "ovmobilebench/" - "scripts/" patch: default: - target: 99% + target: auto threshold: 1% parsers: diff --git a/requirements.txt b/requirements.txt index 37d3399..15de124 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,11 @@ rich>=13.7.1 adbutils>=2.0.0 certifi>=2024.0.0 +# Build dependencies +build>=1.0.0 +setuptools>=70.0.0 +wheel>=0.40.0 + # Development dependencies pytest>=8.2.0 pytest-cov>=5.0.0