optimize ci

.github/generate-envs.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+
+"""Generate a list of test environments.
+
+Each environment must contain the following fields:
+
+Optional fields:
+
+What tests belong in what groups:
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+
+ENVS = [
+    {
+        "python-version": "3.13",
+        "nvc-version": "1.20.1",
+        "os": "ubuntu-24.04",
+        "simulator": "nvc",
+        "toplevel_lang": "vhdl",
+        "cc": "clang",
+        "cxx": "clang++",
+        "gcov": "llvm-cov gcov",
+        "cxx-standard": "20",
+    },
+    {
+        "python-version": "3.13",
+        "simulator": "icarus",
+        "toplevel_lang": "verilog",
+        "os": "macos-26",
+        "cxx-standard": "20",
+    },
+    {
+        "python-version": "3.13",
+        "simulator": "icarus",
+        "toplevel_lang": "verilog",
+        "os": "macos-15-intel",
+        "cxx-standard": "20",
+    },
+    {
+        "python-version": "3.13",
+        "nvc-version": "1.20.1",
+        "os": "ubuntu-24.04",
+        "simulator": "nvc",
+        "toplevel_lang": "vhdl",
+        "cxx-standard": "23",
+    },
+]
+
+# # cpp tests without a simulator
+# ubuntu_versions = ["Ubuntu 22.04", "Ubuntu 24.04"]
+# cpp_versions = ["20", "23"]
+# for ubuntu_ver in ubuntu_versions:
+#     for cpp_ver in cpp_versions:
+#         ENVS += {
+#             "cxx-standard": cpp_ver,
+#             "os": ubuntu_ver,
+#             "group": "cpp-tests-no-sim"
+#         }
+
+# # python tests without a simulator
+# python_versions = ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"]
+# for ver in python_versions:
+#     ENVS += {
+#         "python-version": ver,
+#         "os": "ubuntu-24.04",
+#         "group": "python-tests-no-sim"
+#     }
+
+
+def append_str_val(listref, my_list, key) -> None:
+    if key not in my_list:
+        return
+    listref.append(str(my_list[key]))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--group")
+    parser.add_argument("--output-format", choices=("gha", "json"), default="json")
+    parser.add_argument(
+        "--gha-output-file",
+        type=argparse.FileType("a", encoding="utf-8"),
+        help="The $GITHUB_OUTPUT file.",
+    )
+
+    args = parser.parse_args()
+
+    if args.group is not None and args.group != "":
+        selected_envs = [t for t in ENVS if "group" in t and t["group"] == args.group]
+    else:
+        # Return all tasks if no group is selected.
+        selected_envs = ENVS
+
+    for env in selected_envs:
+        # Assemble the human-readable name of the job.
+        name_parts = []
+        append_str_val(name_parts, env, "extra-name")
+        append_str_val(name_parts, env, "sim")
+        name_parts.append(env["sim-version"])
+        append_str_val(name_parts, env, "lang")
+        append_str_val(name_parts, env, "os")
+        append_str_val(name_parts, env, "python-version")
+        if env.get("may-fail-dev") is not None:
+            name_parts.append("May fail")
+
+        env["name"] = "|".join(name_parts)
+
+    if args.output_format == "gha":
+        assert args.gha_output_file is not None
+
+        print(f"envs={json.dumps(selected_envs)}", file=args.gha_output_file)
+
+        print("Generated the following test configurations:")
+        print(json.dumps(selected_envs, indent=2))
+
+    elif args.output_format == "json":
+        print(json.dumps(selected_envs, indent=2))
+
+    else:
+        assert False
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/regression-tests.yaml

@@ -0,0 +1,107 @@
+name: Regression Tests
+
+on:
+  workflow_call:
+    inputs:
+      test_task:
+        required: true
+        type: string
+        default: dev_test
+      collect_coverage:
+        required: false
+        type: boolean
+        default: false
+
+
+jobs:
+    generate_envs:
+    runs-on: ubuntu-latest
+    name: Generate a list of environments to run tests against
+    steps:
+    - uses: actions/checkout@v6
+    - run: ./.github/generate-envs.py --output-format=gha --gha-output-file="$GITHUB_OUTPUT" --group="${{inputs.group}}"
+      id: run_generate_script
+    outputs:
+      envs: ${{ steps.run_generate_script.outputs.envs }}
+
+    tests:
+      needs: generate_envs
+
+      name: ${{matrix.name}}
+      runs-on: ${{matrix.runs-on}}
+      timeout-minutes: 60
+
+      env:
+        SIM: ${{ matrix.simulator }}
+        TOPLEVEL_LANG: ${{ matrix.toplevel_lang }}
+        CXX_STANDARD: ${{ matrix.cxx-standard }}
+
+      strategy:
+        fail-fast: false
+        matrix:
+          include: ${{ fromJson(needs.generate_envs.outputs.envs) }}
+
+      steps:
+      - uses: actions/checkout@v6
+        with:
+          # GitHub PR's create a merge commit, and Actions are run on that commit.
+          # Codecov's uploader needs the previous commit (tip of PR branch) to associate coverage correctly.
+          # A fetch depth of 2 provides enough information without fetching the entire history.
+          fetch-depth: 2
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: ${{matrix.python-version}}
+          allow-prereleases: true
+
+      - name: Setup NVC
+        uses: nickg/setup-nvc@v1
+        if: matrix.simulator == 'nvc'
+        with:
+          version: ${{matrix.nvc-version}}
+
+      - name: Setup Icarus (MacOS)
+        if: matrix.simulator == 'icarus' && startsWith(matrix.os, 'macos')
+        run: |
+          brew install icarus-verilog
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          activate-environment: "true"
+
+      - name: Install Testing Dependencies
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt install -y make libgtest-dev llvm
+
+      - name: Checkout cocotb
+        uses: actions/checkout@v6
+        with:
+          repository: cocotb/cocotb
+          path: cocotb
+
+      - name: Run tests
+        env:
+          CC: ${{ matrix.cc || 'gcc' }}
+          CXX: ${{ matrix.cxx || 'g++' }}
+          GCOV_EXECUTABLE: ${{ matrix.gcov || 'gcov' }}
+          CXX_STANDARD: ${{ matrix.cxx-standard }}
+          DEV_BUILD_DEP_GROUP: dev_tests
+        run: |
+          make dev_tests integration_tests generate_report
+
+      - name: Check stubs haven't diverged
+        run: |
+          git diff --exit-code -- python/_coconext.pyi
+
+      - name: Upload coverage reports to Codecov
+        if: github.repository == 'ktbarrett/coconext'
+        uses: codecov/codecov-action@v6
+        with:
+          disable_search: true
+          plugins: noop
+          files: .python-coverage.xml,.cpp-coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
+          fail_ci_if_error: true

.github/workflows/test-dev.yaml

@@ -0,0 +1,26 @@
+name: CI
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
+  cancel-in-progress: ${{ !contains(github.ref, 'dev') }}
+
+on:
+  push:
+    branches:
+      - dev
+    paths-ignore:
+      - 'docs/dev/**'
+  pull_request:
+    branches:
+      - dev
+    paths-ignore:
+      - 'docs/dev/**'
+
+
+jobs:
+  test_dev_cpp:
+    name: Regression Tests
+    uses: ./.github/workflows/regression-tests.yml
+    with:
+      test_task: dev_test
+      collect_coverage: true