CompletePipeline.yml

# ==================================================================================================================== #
# Authors:                                                                                                             #
#   Patrick Lehmann                                                                                                    #
#                                                                                                                      #
# ==================================================================================================================== #
# Copyright 2020-2026 The pyTooling Authors                                                                            #
#                                                                                                                      #
# Licensed under the Apache License, Version 2.0 (the "License");                                                      #
# you may not use this file except in compliance with the License.                                                     #
# You may obtain a copy of the License at                                                                              #
#                                                                                                                      #
#   http://www.apache.org/licenses/LICENSE-2.0                                                                         #
#                                                                                                                      #
# Unless required by applicable law or agreed to in writing, software                                                  #
# distributed under the License is distributed on an "AS IS" BASIS,                                                    #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.                                             #
# See the License for the specific language governing permissions and                                                  #
# limitations under the License.                                                                                       #
#                                                                                                                      #
# SPDX-License-Identifier: Apache-2.0                                                                                  #
# ==================================================================================================================== #
name: Complete Pipeline

on:
  workflow_call:
    inputs:
      package_namespace:
        description: 'Name of the tool''s namespace.'
        required: false
        default: ''
        type: string
      package_name:
        description: 'Name of the tool''s package.'
        required: true
        type: string
      unittest_python_version:
        description: 'Python version.'
        required: false
        default: '3.14'
        type: string
      unittest_python_version_list:
        description: 'Space separated list of Python versions to run tests with.'
        required: false
        default: '3.10 3.11 3.12 3.13 3.14'
        type: string
      unittest_system_list:
        description: 'Space separated list of systems to run tests on.'
        required: false
        default: 'ubuntu ubuntu-arm windows windows-arm macos macos-arm mingw64 ucrt64'
        type: string
      unittest_include_list:
        description: 'Space separated list of system:python items to be included into the list of test.'
        required: false
        default: ''
        type: string
      unittest_exclude_list:
        description: 'Space separated list of system:python items to be excluded from the list of test.'
        required: false
        default: 'windows-arm:3.9 windows-arm:3.10'
        type: string
      unittest_disable_list:
        description: 'Space separated list of system:python items to be disabled from the list of test.'
        required: false
        default: 'windows-arm:pypy-3.10 windows-arm:pypy-3.11'
        type: string
      apptest:
        description: 'Run Application Testing.'
        required: false
        default: 'false'
        type: string
      apptest_python_version:
        description: 'Python version.'
        required: false
        default: '3.14'
        type: string
      apptest_python_version_list:
        description: 'Space separated list of Python versions to run tests with.'
        required: false
        default: ''
        type: string
      apptest_system_list:
        description: 'Space separated list of systems to run tests on.'
        required: false
        default: 'ubuntu ubuntu-arm windows windows-arm macos macos-arm ucrt64'
        type: string
      apptest_include_list:
        description: 'Space separated list of system:python items to be included into the list of test.'
        required: false
        default: ''
        type: string
      apptest_exclude_list:
        description: 'Space separated list of system:python items to be excluded from the list of test.'
        required: false
        default: 'windows-arm:3.9 windows-arm:3.10'
        type: string
      apptest_disable_list:
        description: 'Space separated list of system:python items to be disabled from the list of test.'
        required: false
        default: 'windows-arm:pypy-3.10 windows-arm:pypy-3.11'
        type: string
      bandit:
        description: 'Run Static Application Security Testing (SAST) using Bandit.'
        required: false
        default: 'false'
        type: string
      pylint:
        description: 'Run Python linting using pylint.'
        required: false
        default: 'false'
        type: string
      codecov:
        description: 'Publish merged coverage and unittest reports to Codecov.'
        required: false
        default: 'false'
        type: string
      codacy:
        description: 'Publish merged coverage report to Codacy.'
        required: false
        default: 'false'
        type: string
      dorny:
        description: 'Publish merged unittest report via Dorny Test-Reporter.'
        required: false
        default: 'false'
        type: string
      documentation_steps:
        description: 'Create documentation.'
        required: false
        default: 'html pages'   # none, html, latex, pdf, pages, asset, all
        type: string
      miktex_image:
        description: 'Name of MiKTeX docker image.'
        required: false
        default: 'pytooling/miktex:sphinx'
        type: string
      miktex_update:
        description: 'Update MiKTeX packages.'
        required: false
        default: 'false'
        type: string
      auto_tag:
        description: 'Auto tag if a branch was merged to the release branch and an associated pull-request with matching version name was found.'
        required: false
        default: 'true'
        type: string
      cleanup:
        description: 'Cleanup artifacts afterwards.'
        required: false
        default: 'true'
        type: string
    secrets:
      PYPI_TOKEN:
        description: "Token for pushing releases to PyPI."
        required: false
      CODECOV_TOKEN:
        description: "Token for pushing coverage and unittest results to Codecov."
        required: false
      CODACY_TOKEN:
        description: "Token for pushing coverage results to Codacy."
        required: false

jobs:
  Prepare:
    uses: pyTooling/Actions/.github/workflows/PrepareJob.yml@dev
    with:
      pipeline-delay: 10

  ConfigParams:
    uses: pyTooling/Actions/.github/workflows/ExtractConfiguration.yml@dev

  UnitTestingParams:
    uses: pyTooling/Actions/.github/workflows/Parameters.yml@dev
    with:
      package_namespace:   ${{ inputs.package_namespace }}
      package_name:        ${{ inputs.package_name }}
      python_version:      ${{ inputs.unittest_python_version }}
      python_version_list: ${{ inputs.unittest_python_version_list }}
      system_list:         ${{ inputs.unittest_system_list }}
      include_list:        ${{ inputs.unittest_include_list }}
      exclude_list:        ${{ inputs.unittest_exclude_list }}
      disable_list:        ${{ inputs.unittest_disable_list }}
      documentation_steps: ${{ inputs.documentation_steps }}

  AppTestingParams:
    uses: pyTooling/Actions/.github/workflows/Parameters.yml@dev
    if: inputs.apptest == 'true'
    with:
      package_namespace:   ${{ inputs.package_namespace }}
      package_name:        ${{ inputs.package_name }}
      python_version:      ${{ inputs.apptest_python_version }}
      python_version_list: ${{ inputs.apptest_python_version_list }}
      system_list:         ${{ inputs.apptest_system_list }}
      include_list:        ${{ inputs.apptest_include_list }}
      exclude_list:        ${{ inputs.apptest_exclude_list }}
      disable_list:        ${{ inputs.apptest_disable_list }}

  InstallParams:
    uses: pyTooling/Actions/.github/workflows/Parameters.yml@dev
    with:
      package_namespace:   ${{ inputs.package_namespace }}
      package_name:        ${{ inputs.package_name }}
      python_version:      ${{ inputs.unittest_python_version }}
      python_version_list: ''
      system_list:         ${{ inputs.unittest_system_list }}
      include_list:        ${{ inputs.unittest_include_list }}
      exclude_list:        ${{ inputs.unittest_exclude_list }}
      disable_list:        ${{ inputs.unittest_disable_list }}
      documentation_steps: 'none'

  VersionCheck:
    runs-on: 'ubuntu-26.04'
    needs:
      - Prepare
      - UnitTestingParams
    if: needs.Prepare.outputs.version != '' && needs.UnitTestingParams.outputs.package_version_file != ''
    outputs:
      code_version: ${{ steps.extract.outputs.code_version }}
    steps:
      - name: ⏬ Checkout repository
        uses: actions/checkout@v7
        with:
          # The command 'git describe' (used for version) needs the history.
          fetch-depth: 0

      - name: 🔧 Install pyTooling dependencies (native)
        run: |
          python -m pip install --disable-pip-version-check -U pyTooling

      - name: Extract version from Python source file
        id: extract
        if: endsWith(needs.UnitTestingParams.outputs.package_version_file, '.py')
        shell: python
        run: |
          from os                   import getenv
          from pathlib              import Path
          from sys                  import exit
          from textwrap             import dedent
          from pyTooling.Packaging  import extractVersionInformation
          from pyTooling.Versioning import SemanticVersion

          expectedVersion = "${{ needs.Prepare.outputs.version }}".strip()

          versionFile = Path("${{ needs.UnitTestingParams.outputs.package_version_file }}")
          if not versionFile.exists():
            print(f"::error title=CompletePipeline::Version file '{versionFile}' doesn't exist.")
            exit(1)

          versionInformation = extractVersionInformation(versionFile)
          print(f"expected:  {expectedVersion}")
          print(f"from code: {versionInformation.Version}")

          # The expected version is derived from the tag, so it may carry a 'v' or 'r' prefix, while
          # a version in Python code never does. Compare the parsed versions, which ignore the prefix.
          try:
            expected = SemanticVersion.Parse(expectedVersion)
            fromCode = SemanticVersion.Parse(versionInformation.Version)
          except Exception as ex:
            print(f"::error title=CompletePipeline::Can't parse version ('{expectedVersion}' / '{versionInformation.Version}'): {ex}")
            exit(2)

          if expected != fromCode:
            print(f"::error title=CompletePipeline::Expected version ({expectedVersion}) doesn't match the version in Python code ({versionInformation.Version}).")
            exit(2)

          # Write jobs to special file
          github_output = Path(getenv("GITHUB_OUTPUT"))
          print(f"GITHUB_OUTPUT: {github_output}")
          with github_output.open("a+", encoding="utf-8") as f:
            f.write(dedent(f"""\
              code_version={versionInformation.Version}
            """))

  UnitTesting:
    uses: pyTooling/Actions/.github/workflows/UnitTesting.yml@dev
    needs:
      - ConfigParams
      - UnitTestingParams
    with:
      jobs: ${{ needs.UnitTestingParams.outputs.python_jobs }}
# TODO: shouldn't this be configured by a parameter? Same as directories
      unittest_report_xml:      ${{ needs.ConfigParams.outputs.unittest_report_xml }}
      coverage_report_xml:      ${{ needs.ConfigParams.outputs.coverage_report_xml }}
      coverage_report_json:     ${{ needs.ConfigParams.outputs.coverage_report_json }}
      coverage_report_html:     ${{ needs.ConfigParams.outputs.coverage_report_html }}
      unittest_xml_artifact:    ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).unittesting_xml }}
      coverage_sqlite_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).codecoverage_sqlite }}

  StaticTypeCheck:
    uses: pyTooling/Actions/.github/workflows/StaticTypeCheck.yml@dev
    needs:
      - ConfigParams
      - UnitTestingParams
    with:
      python_version:     ${{ needs.UnitTestingParams.outputs.python_version }}
      cobertura_report:   ${{ needs.ConfigParams.outputs.typing_report_cobertura }}
      junit_report:       ${{ needs.ConfigParams.outputs.typing_report_junit }}
      html_report:        ${{ needs.ConfigParams.outputs.typing_report_html }}
      cobertura_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_cobertura }}
      junit_artifact:     ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_junit }}
      html_artifact:      ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_html }}

  CodeQuality:
    uses: pyTooling/Actions/.github/workflows/CheckCodeQuality.yml@dev
    needs:
      - UnitTestingParams
      - PublishTestResults   # artificial dependency to delay start when pipeline has free job resources
    if: ${{ !cancelled() }}  # the artificial dependency above is for ordering only and must not gate this job
    with:
      python_version:    ${{ needs.UnitTestingParams.outputs.python_version }}
      package_directory: ${{ needs.UnitTestingParams.outputs.package_directory }}
      bandit:            ${{ inputs.bandit }}
      pylint:            ${{ inputs.pylint }}
      artifact:          CodeQuality

  DocCoverage:
    uses: pyTooling/Actions/.github/workflows/CheckDocumentation.yml@dev
    needs:
      - UnitTestingParams
      - StaticTypeCheck      # artificial dependency to delay start when pipeline has free job resources
    if: ${{ !cancelled() }}  # the artificial dependency above is for ordering only and must not gate this job
    with:
      python_version: ${{ needs.UnitTestingParams.outputs.python_version }}
      directory:      ${{ needs.UnitTestingParams.outputs.package_directory }}

  Package:
    uses: pyTooling/Actions/.github/workflows/Package.yml@dev
    needs:
      - UnitTestingParams
    with:
      python_version: ${{ needs.UnitTestingParams.outputs.python_version }}
      artifact:       ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).package_all }}

  Install:
    uses: pyTooling/Actions/.github/workflows/InstallPackage.yml@dev
    needs:
      - UnitTestingParams
      - InstallParams
      - Package
    with:
      jobs:         ${{ needs.InstallParams.outputs.python_jobs }}
      wheel:        ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).package_all }}
      package_name: ${{ needs.UnitTestingParams.outputs.package_fullname }}

  AppTesting:
    uses: pyTooling/Actions/.github/workflows/ApplicationTesting.yml@dev
    needs:
      - ConfigParams
      - AppTestingParams
      - UnitTestingParams
      - Package
    if: inputs.apptest == 'true'
    with:
      jobs:                 ${{ needs.AppTestingParams.outputs.python_jobs }}
      wheel:                ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).package_all }}
      apptest_report_xml:   ${{ needs.ConfigParams.outputs.unittest_report_xml }}
      apptest_xml_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).apptesting_xml }}

  PublishCoverageResults:
    uses: pyTooling/Actions/.github/workflows/PublishCoverageResults.yml@dev
    needs:
      - ConfigParams
      - UnitTestingParams
      - UnitTesting
    if: ${{ !cancelled() }}  # not 'success() || failure()', because that's false if a dependency was skipped
    with:
#      coverage_sqlite_artifact:       ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).codecoverage_sqlite }}
#      coverage_xml_artifact:          ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).codecoverage_xml }}
#      coverage_report_xml_directory:  ${{ needs.ConfigParams.outputs.coverage_report_xml_directory }}
#      coverage_report_xml_filename:   ${{ needs.ConfigParams.outputs.coverage_report_xml_filename }}
      coverage_report_json:   ${{ needs.ConfigParams.outputs.coverage_report_json }}
      coverage_report_html:   ${{ needs.ConfigParams.outputs.coverage_report_html }}
      coverage_json_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).codecoverage_json }}
      coverage_html_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).codecoverage_html }}
      codecov:                ${{ inputs.codecov }}
      codacy:                 ${{ inputs.codacy }}
    secrets:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
      CODACY_TOKEN:  ${{ secrets.CODACY_TOKEN }}

  PublishTestResults:
    uses: pyTooling/Actions/.github/workflows/PublishTestResults.yml@dev
    needs:
      - ConfigParams
      - UnitTestingParams
      - UnitTesting
      - AppTesting
    if: ${{ !cancelled() }}  # not 'success() || failure()', because that's false if AppTesting was skipped (apptest: false)
    with:
      testsuite-summary-name: ${{ needs.UnitTestingParams.outputs.package_fullname }}
      additional_merge_args: '"--pytest=rewrite-dunder-init;reduce-depth:pytest.tests"'
      merged_junit_filename:  ${{ fromJson(needs.ConfigParams.outputs.unittest_merged_report_xml).filename }}
      merged_junit_artifact:  ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).unittesting_xml }}
      dorny:                  ${{ inputs.dorny }}
      codecov:                ${{ inputs.codecov }}
    secrets:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

#  VerifyDocs:
#    uses: pyTooling/Actions/.github/workflows/VerifyDocs.yml@dev
#    needs:
#      - UnitTestingParams
#    with:
#      python_version: ${{ needs.UnitTestingParams.outputs.python_version }}

  Documentation:
    uses: pyTooling/Actions/.github/workflows/SphinxDocumentation.yml@dev
    needs:
      - ConfigParams
      - UnitTestingParams
      - PublishTestResults
      - PublishCoverageResults
#      - VerifyDocs
    if: ${{ !cancelled() }}  # not 'success() || failure()', because that's false if a dependency was skipped
    with:
      python_version:         ${{ needs.UnitTestingParams.outputs.python_version }}
      coverage_report_json:   ${{ needs.ConfigParams.outputs.coverage_report_json }}
      unittest_xml_artifact:  ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).unittesting_xml }}
      coverage_json_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).codecoverage_json }}
      html_artifact:          ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).documentation_html }}
      latex_artifact:         ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).documentation_latex }}

  IntermediateCleanUp:
    uses: pyTooling/Actions/.github/workflows/CleanupArtifacts.yml@dev
    needs:
      - UnitTestingParams
      - PublishCoverageResults
      - PublishTestResults
    # not 'success() || failure()', because that's false if a dependency was skipped
    if: >-
      ${{ !cancelled()
       && inputs.cleanup == 'true'
      }}
    with:
      json: ${{ needs.UnitTestingParams.outputs.artifact_names }}
      artifact-json-ids: >-
        unittesting_xml:-*
        codecoverage_sqlite:-*

  PDFDocumentation:
    uses: pyTooling/Actions/.github/workflows/LaTeXDocumentation.yml@dev
    needs:
      - UnitTestingParams
      - Documentation
    if: >-
      ${{ !failure() && !cancelled()
       && needs.Documentation.result == 'success'
       && contains(inputs.documentation_steps, 'latex')
       && contains(inputs.documentation_steps, 'pdf')
      }}
    with:
      document:       ${{ needs.UnitTestingParams.outputs.package_fullname }}
      latex_artifact: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).documentation_latex }}
      pdf_artifact:   ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).documentation_pdf }}
      can-fail:       'true'
      miktex_image:   ${{ inputs.miktex_image }}
      update:         ${{ inputs.miktex_update }}

  PublishToGitHubPages:
    uses: pyTooling/Actions/.github/workflows/PublishToGitHubPages.yml@dev
    needs:
      - UnitTestingParams
      - Documentation
#      - PDFDocumentation
      - PublishCoverageResults
      - StaticTypeCheck
    if: >-
      ${{ !failure() && !cancelled()
       && needs.Documentation.result == 'success'
       && needs.PublishCoverageResults.result == 'success'
       && needs.StaticTypeCheck.result == 'success'
       && contains(inputs.documentation_steps, 'pages')
      }}
    with:
      doc:      ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).documentation_html }}
      coverage: ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).codecoverage_html }}
      typing:   ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).statictyping_html }}
      cleanup:  ${{ inputs.cleanup }}

  TriggerTaggedRelease:
    uses: pyTooling/Actions/.github/workflows/TagReleaseCommit.yml@dev
    needs:
      - Prepare
      - UnitTesting
#      - AppTesting
#      - StaticTypeCheck
      - Package
      - Install
      - PublishToGitHubPages
    if: >-
      ${{ !failure() && !cancelled()
       && needs.Prepare.outputs.is_release_commit == 'true'
       && github.event_name != 'schedule'
      }}
    permissions:
      contents: write  # required for create tag
      actions:  write  # required for trigger workflow
    with:
      version:  ${{ needs.Prepare.outputs.version }}
      auto_tag: ${{ needs.Prepare.outputs.is_release_commit == 'true' && inputs.auto_tag == 'true' }}
    secrets: inherit

  ReleasePage:
    uses: pyTooling/Actions/.github/workflows/PublishReleaseNotes.yml@dev
    needs:
      - Prepare
      - UnitTesting
#      - AppTesting
#      - StaticTypeCheck
      - Package
      - Install
      - PublishToGitHubPages
    if: >-
      ${{ !failure() && !cancelled()
       && needs.Prepare.outputs.is_release_tag == 'true'
      }}
    permissions:
      contents: write
      actions:  write
    with:
      tag: ${{ needs.Prepare.outputs.version }}
    secrets: inherit

  PublishOnPyPI:
    uses: pyTooling/Actions/.github/workflows/PublishOnPyPI.yml@dev
    needs:
      - Prepare
      - UnitTestingParams
      - Package
      - ReleasePage
    if: >-
      ${{ !failure() && !cancelled()
       && needs.Package.result == 'success'
       && needs.Prepare.outputs.is_release_tag == 'true'
      }}
    with:
      python_version: ${{ needs.UnitTestingParams.outputs.python_version }}
      requirements:   '-r dist/requirements.txt'
      artifact:       ${{ fromJson(needs.UnitTestingParams.outputs.artifact_names).package_all }}
      cleanup:        ${{ inputs.cleanup }}
    secrets:
      PYPI_TOKEN:     ${{ secrets.PYPI_TOKEN }}

  ArtifactCleanUp:
    uses: pyTooling/Actions/.github/workflows/CleanupArtifacts.yml@dev
    needs:
      - Prepare
      - UnitTestingParams
      - UnitTesting
      - StaticTypeCheck
      - Documentation
      - PDFDocumentation
      - PublishTestResults
      - PublishCoverageResults
      - PublishToGitHubPages
#      - PublishOnPyPI
      - Install
      - IntermediateCleanUp
    # !cancelled(), because PDFDocumentation is skipped without 'latex' and 'pdf' in documentation_steps
    if: >-
      ${{ !cancelled()
       && inputs.cleanup == 'true'
      }}
    with:
      json: ${{ needs.UnitTestingParams.outputs.artifact_names }}
      artifact-json-ids: >-
        unittesting_html:-*
        codecoverage_xml:-*
        codecoverage_json:-*
        codecoverage_html:-*
        unittesting_xml
        unittesting_html
        codecoverage_sqlite
        codecoverage_xml
        codecoverage_json
        codecoverage_html
        statictyping_html
        #apptesting_xml:-*
        documentation_html
        #documentation_latex
        #documentation_pdf
      json2:      ${{ needs.UnitTestingParams.outputs.artifact_names }}
      condition2: ${{ needs.Prepare.outputs.is_release_tag != 'true' }}  # the same flag 'PublishOnPyPI' keys off: it consumes the package artifact and deletes it itself
      artifact-json-ids2: >-
        package_all