From 7c93a0df3c4dcc18854cc5a4c938d5efe7136cda Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 11:41:36 -0600 Subject: [PATCH 01/13] ci: simplify GitHub workflow setup and labeling - add shared setup-reeds-env composite action for Python/Julia setup\n- refactor CI workflow to reuse setup action and remove cache workflow\n- add dependabot, labeler, and workflow-quality configs\n- align composite action metadata and workflow shell quoting for linting --- .github/actions/r2x-check/action.yaml | 9 +-- .github/actions/setup-reeds-env/action.yml | 72 +++++++++++++++++ .github/dependabot.yml | 65 +++++++++++++++ .github/labeler.yaml | 92 +++++++++++++++++++++ .github/workflows/build-docs.yaml | 8 +- .github/workflows/cache.yaml | 67 ---------------- .github/workflows/labeler.yaml | 27 +++++++ .github/workflows/python-app.yaml | 93 ++++++---------------- .github/workflows/workflow-quality.yaml | 50 ++++++++++++ 9 files changed, 336 insertions(+), 147 deletions(-) create mode 100644 .github/actions/setup-reeds-env/action.yml create mode 100644 .github/dependabot.yml create mode 100644 .github/labeler.yaml delete mode 100644 .github/workflows/cache.yaml create mode 100644 .github/workflows/labeler.yaml create mode 100644 .github/workflows/workflow-quality.yaml diff --git a/.github/actions/r2x-check/action.yaml b/.github/actions/r2x-check/action.yaml index 8b726751..b3025c34 100644 --- a/.github/actions/r2x-check/action.yaml +++ b/.github/actions/r2x-check/action.yaml @@ -3,18 +3,13 @@ description: "Action to run R2X." inputs: reeds-run-path: required: true - type: string scenario: - type: string python-version: - type: string default: "3.11" weather-year: - type: int - default: 2012 + default: "2012" solve-year: - type: int - default: 2035 + default: "2035" outputs: scenario: diff --git a/.github/actions/setup-reeds-env/action.yml b/.github/actions/setup-reeds-env/action.yml new file mode 100644 index 00000000..d514763d --- /dev/null +++ b/.github/actions/setup-reeds-env/action.yml @@ -0,0 +1,72 @@ +name: Setup ReEDS Environment +description: Setup cached micromamba Python and/or Julia environment for ReEDS CI. + +inputs: + setup-python: + description: Setup micromamba environment from environment.yml. + required: false + default: "true" + setup-julia: + description: Setup Julia toolchain. + required: false + default: "true" + julia-version: + description: Julia version to install. + required: false + default: "1.12.1" + instantiate-julia: + description: Run julia --project=. instantiate.jl. + required: false + default: "true" + remove-conda-julia: + description: Remove julia from micromamba env before setup-julia. + required: false + default: "true" + +outputs: + mamba-env-path: + description: Micromamba environment path. + value: ${{ steps.mamba.outputs.environment-path }} + julia-bindir: + description: Julia binary directory. + value: ${{ steps.setup-julia.outputs.julia-bindir }} + +runs: + using: composite + steps: + - name: Setup micromamba env + if: ${{ fromJSON(inputs.setup-python) }} + id: mamba + uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2 + with: + environment-file: environment.yml + cache-environment: true + post-cleanup: "all" + cache-environment-key: ${{ runner.os }}-conda-${{ hashFiles('environment.yml') }} + + - name: Remove conda Julia package + if: ${{ fromJSON(inputs.setup-python) && fromJSON(inputs.setup-julia) && fromJSON(inputs.remove-conda-julia) }} + shell: bash -le {0} + run: micromamba remove -y julia || true + + - name: Restore Julia cache + if: ${{ fromJSON(inputs.setup-julia) }} + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 + with: + path: ~/.julia + key: julia-${{ runner.os }}-${{ hashFiles('Project.toml') }} + restore-keys: julia-${{ runner.os }}- + + - name: Setup Julia + if: ${{ fromJSON(inputs.setup-julia) }} + id: setup-julia + uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2 + with: + version: ${{ inputs.julia-version }} + env: + NODE_EXTRA_CA_CERTS: /usr/local/share/ca-certificates/nrel_ca_chain.crt + + - name: Instantiate Julia project + if: ${{ fromJSON(inputs.setup-julia) && fromJSON(inputs.instantiate-julia) }} + shell: bash -le {0} + run: julia --project=. instantiate.jl diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..c4a05ef9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,65 @@ +version: 2 + +updates: + - package-ecosystem: "conda" + directory: / + schedule: + interval: "weekly" + cooldown: + default-days: 7 + open-pull-requests-limit: 2 + labels: + - "dependenabot" + reviewers: + - "kennedy-mindermann" + commit-message: + prefix: "build" + include: "scope" + rebase-strategy: "auto" + + - package-ecosystem: "pip" + directory: / + schedule: + interval: "weekly" + cooldown: + default-days: 7 + open-pull-requests-limit: 2 + labels: + - "dependenabot" + reviewers: + - "kennedy-mindermann" + commit-message: + prefix: "build" + include: "scope" + rebase-strategy: "auto" + + - package-ecosystem: "julia" + directory: / + schedule: + interval: "weekly" + cooldown: + default-days: 7 + open-pull-requests-limit: 2 + labels: + - "dependenabot" + reviewers: + - "kennedy-mindermann" + commit-message: + prefix: "build" + include: "scope" + rebase-strategy: "auto" + + - package-ecosystem: "github-actions" + directory: / + schedule: + interval: "weekly" + cooldown: + default-days: 7 + labels: + - "dependenabot" + reviewers: + - "kennedy-mindermann" + commit-message: + prefix: "build" + include: "scope" + rebase-strategy: "auto" diff --git a/.github/labeler.yaml b/.github/labeler.yaml new file mode 100644 index 00000000..198cf14d --- /dev/null +++ b/.github/labeler.yaml @@ -0,0 +1,92 @@ +dependenabot: + - head-branch: + - '^dependabot/.+' + +dependencies: + - changed-files: + - any-glob-to-any-file: + - "environment.yml" + - "pyproject.toml" + - "requirements*.txt" + - "Project.toml" + - "Manifest.toml" + - ".github/dependabot.yml" + +github_actions: + - changed-files: + - any-glob-to-any-file: + - ".github/workflows/**" + - ".github/actions/**" + - ".github/scripts/**" + +docs: + - changed-files: + - any-glob-to-any-file: + - "docs/**" + - "**/*.md" + - "CITATION.cff" + +release: + - changed-files: + - any-glob-to-any-file: + - "CHANGELOG.md" + - "README.md" + +tests: + - changed-files: + - any-glob-to-any-file: + - "tests/**" + - "hourlize/tests/**" + - "reeds2pras/test/**" + - "**/test_*.py" + - "**/*_test.py" + +model_changes: + - changed-files: + - any-glob-to-any-file: + - "**/*.gms" + - "runbatch.py" + - "d_solve*.py" + - "createmodel.gms" + - "reeds/**" + - "ReEDS_Augur/**" + - "Augur.py" + - "instantiate.jl" + - "reeds2pras/**" + +data_changes: + - changed-files: + - any-glob-to-any-file: + - "inputs/**" + - "hourlize/inputs/**" + - "postprocessing/**/inputs/**" + - "**/*.csv" + - "**/*.h5" + - "**/*.hdf5" + - "**/*.parquet" + - "**/*.xlsx" + - "**/*.xls" + - "sources.csv" + - "sources_documentation.md" + +hourlize: + - changed-files: + - any-glob-to-any-file: "hourlize/**" + +input_processing: + - changed-files: + - any-glob-to-any-file: + - "input_processing/**" + - "preprocessing/**" + +postprocessing: + - changed-files: + - any-glob-to-any-file: "postprocessing/**" + +reeds2pras: + - changed-files: + - any-glob-to-any-file: "reeds2pras/**" + +inputs: + - changed-files: + - any-glob-to-any-file: "inputs/**" diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index fac37fb5..19b447ab 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -48,11 +48,11 @@ jobs: - name: Set variables for internal github repo run: | - echo "BASE_URL=https://github.com/ReEDS-Model/ReEDS" >> $GITHUB_ENV - cd ${GITHUB_WORKSPACE}/docs/source/documentation_tools/ + echo "BASE_URL=https://github.com/ReEDS-Model/ReEDS" >> "$GITHUB_ENV" + cd "$GITHUB_WORKSPACE/docs/source/documentation_tools/" sh generate_sources_md_file.sh - cd ${GITHUB_WORKSPACE} - python docs/source/documentation_tools/generate_markdown.py --githubURL "https://github.com/ReEDS-Model/ReEDS/blob/main" --reedsPath "${GITHUB_WORKSPACE}" + cd "$GITHUB_WORKSPACE" + python docs/source/documentation_tools/generate_markdown.py --githubURL "https://github.com/ReEDS-Model/ReEDS/blob/main" --reedsPath "$GITHUB_WORKSPACE" - name: Build Sphinx documentation env: diff --git a/.github/workflows/cache.yaml b/.github/workflows/cache.yaml deleted file mode 100644 index b44aa9f3..00000000 --- a/.github/workflows/cache.yaml +++ /dev/null @@ -1,67 +0,0 @@ -# This Workflow creates caches for the Python and Julia environments every time -# someone merges a PR. Subsequent PRs reuse these caches if the dependency -# files have not changed. -name: update-cache - -on: - push: - branches: - - main - workflow_dispatch: - -permissions: - contents: read - -# To reset the CACHE, change the environment variable: CACHE_NUMBER. -# Either you increment the number or decrement it, but also reflect the changes -# on the other workflows that reutilize the Python environment -env: - CACHE_NUMBER: 1 - JULIA_VERSION: '1.12.1' - -jobs: - python-cache: - name: Python Cache - runs-on: ubuntu-latest - permissions: - contents: read - actions: write - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2 - with: - environment-file: environment.yml - cache-environment: true - post-cleanup: 'all' - cache-environment-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} - env: - ACTIONS_STEP_DEBUG: true - - julia-cache: - name: Julia Cache - runs-on: ubuntu-latest - permissions: - contents: read - actions: write - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Restore Julia cache - id: julia-cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 - with: - path: ~/.julia - key: julia-${{ runner.os }}-${{ hashFiles('Project.toml') }} - restore-keys: julia-${{ runner.os }}- - - - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2 - with: - version: ${{ env.JULIA_VERSION }} - env: - NODE_EXTRA_CA_CERTS: /usr/local/share/ca-certificates/nrel_ca_chain.crt - - - run: julia --project=. instantiate.jl diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml new file mode 100644 index 00000000..8acae743 --- /dev/null +++ b/.github/workflows/labeler.yaml @@ -0,0 +1,27 @@ +name: labeler + +on: + pull_request_target: + types: [opened, reopened, synchronize, ready_for_review] + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + pull-requests: write + issues: write + +jobs: + labeler: + name: apply labels + if: github.event.pull_request.draft == false + timeout-minutes: 5 + runs-on: ubuntu-latest + steps: + - uses: actions/labeler@634933edcd8ababfe52f92936142cc22ac488b1b # v6.0.1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + configuration-path: .github/labeler.yaml + sync-labels: true diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 69efc6ff..b851597d 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -23,35 +23,13 @@ defaults: shell: bash -le {0} env: - CACHE_NUMBER: 1 JULIA_VERSION: '1.12.1' - REEDS_ENV_NAME: reeds2 jobs: - julia-setup: - name: "Julia environment" - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 - with: - persist-credentials: false - - - name: Restore Julia cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 - with: - path: ~/.julia - key: julia-${{ runner.os }}-${{ hashFiles('Project.toml') }} - restore-keys: julia-${{ runner.os }}- - - - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2 - with: - version: ${{ env.JULIA_VERSION }} - - - run: julia --project=. instantiate.jl - - python-setup: - name: "Python environment" + env-prime: + name: Prime Python and Julia caches runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read actions: write @@ -59,18 +37,17 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: persist-credentials: false - - uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2 + - name: Setup ReEDS environments + uses: ./.github/actions/setup-reeds-env with: - environment-file: environment.yml - cache-environment: true - post-cleanup: 'all' - cache-environment-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} + julia-version: ${{ env.JULIA_VERSION }} zenodo-setup: name: "Get Zenodo files" needs: - - python-setup + - env-prime runs-on: ubuntu-latest + timeout-minutes: 15 permissions: contents: read actions: write @@ -104,13 +81,10 @@ jobs: zenodo-files-${{ steps.zenodo-files.outputs.files-hash }}- zenodo-files- - - uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2 - id: mamba + - name: Setup Python environment + uses: ./.github/actions/setup-reeds-env with: - environment-file: environment.yml - cache-environment: true - post-cleanup: 'all' - cache-environment-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} + setup-julia: false - name: Download Zenodo files if not cached if: steps.cache-zenodo.outputs.cache-hit != 'true' @@ -118,9 +92,9 @@ jobs: run-ReEDS: runs-on: ubuntu-latest + timeout-minutes: 60 needs: - - julia-setup - - python-setup + - env-prime - zenodo-setup permissions: contents: read @@ -186,46 +160,26 @@ jobs: zenodo-files-${{ steps.zenodo-files.outputs.files-hash }}- zenodo-files- - - uses: mamba-org/setup-micromamba@add3a49764cedee8ee24e82dfde87f5bc2914462 # v2 - id: mamba - with: - environment-file: environment.yml - cache-environment: true - post-cleanup: 'all' - cache-environment-key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles('environment.yml') }} - - - name: Removing julia from python - run: micromamba remove -y julia - - - uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2 - id: setup-julia + - name: Setup Python and Julia environments + id: setup-env + uses: ./.github/actions/setup-reeds-env with: - version: ${{ env.JULIA_VERSION }} - - - name: Restore Julia cache - uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 - with: - path: ~/.julia - key: julia-${{ runner.os }}-${{ hashFiles('Project.toml') }} - restore-keys: julia-${{ runner.os }}- - - - run: julia --project=. instantiate.jl + julia-version: ${{ env.JULIA_VERSION }} - name: Prepend Julia install to PATH - env: - JULIA_BINDIR: ${{ steps.setup-julia.outputs.julia-bindir }} - run: echo "$JULIA_BINDIR" >> $GITHUB_PATH + run: echo "${{ steps.setup-env.outputs.julia-bindir }}" >> "$GITHUB_PATH" - name: Setup environment variables run: | - echo "GAMSDIR=$(python -c 'from gamspy_base import directory;print(directory)')" >> "$GITHUB_ENV" - echo "$(python -c 'from gamspy_base import directory;print(directory)')" >> "$GITHUB_PATH" + GAMSDIR="$(python -c 'from gamspy_base import directory;print(directory)')" + echo "GAMSDIR=$GAMSDIR" >> "$GITHUB_ENV" + echo "$GAMSDIR" >> "$GITHUB_PATH" echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GAMSDIR" >> "$GITHUB_ENV" - name: Setup licence env: GAMSLICE_CONTENT: ${{ secrets.GAMSLICE }} - MAMBA_ENV_PATH: ${{ steps.mamba.outputs.environment-path }} + MAMBA_ENV_PATH: ${{ steps.setup-env.outputs.mamba-env-path }} run: | echo "$GAMSLICE_CONTENT" > "${MAMBA_ENV_PATH}/lib/python3.11/site-packages/gamspy_base/gamslice.txt" @@ -238,7 +192,7 @@ jobs: - name: Print GAMS log if: runner.debug == '1' - run: cat $RUN_FOLDER/gamslog.txt + run: cat "$RUN_FOLDER/gamslog.txt" - name: Check ReEDS outputs if: ${{ hashFiles(format('{0}/outputs/cap_ivrt.csv', env.RUN_FOLDER)) == '' }} @@ -267,6 +221,7 @@ jobs: run-R2X: runs-on: ubuntu-latest + timeout-minutes: 30 needs: - run-ReEDS permissions: diff --git a/.github/workflows/workflow-quality.yaml b/.github/workflows/workflow-quality.yaml new file mode 100644 index 00000000..b71d455f --- /dev/null +++ b/.github/workflows/workflow-quality.yaml @@ -0,0 +1,50 @@ +name: GitHub Actions Quality + +on: + pull_request: + paths: + - ".github/**" + push: + branches: + - main + paths: + - ".github/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + zizmor: + name: zizmor + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Install uv + uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 + with: + version: "0.9.4" + + - name: Run zizmor + run: uvx zizmor .github + + actionlint: + name: actionlint + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + persist-credentials: false + + - name: Run actionlint + run: | + bash <(curl -s https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + ./actionlint From c7aa81d0d4ba3e11ed2f70b0a9a223bda3fbbbdb Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 11:45:16 -0600 Subject: [PATCH 02/13] fix(ci): resolve workflow quality failures - switch labeler trigger to pull_request to satisfy zizmor\n- pin actions/cache refs in CI workflow\n- harden PATH export step against template injection warning\n- set persist-credentials: false in docs checkout --- .github/workflows/build-docs.yaml | 1 + .github/workflows/labeler.yaml | 2 +- .github/workflows/python-app.yaml | 8 +++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index 19b447ab..d1722783 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -27,6 +27,7 @@ jobs: with: fetch-depth: 1 lfs: true + persist-credentials: false - name: Set up Python 3.12 uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml index 8acae743..324fb6d0 100644 --- a/.github/workflows/labeler.yaml +++ b/.github/workflows/labeler.yaml @@ -1,7 +1,7 @@ name: labeler on: - pull_request_target: + pull_request: types: [opened, reopened, synchronize, ready_for_review] concurrency: diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index b851597d..e58a3af6 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -68,7 +68,7 @@ jobs: - name: Restore Zenodo files from cache id: cache-zenodo - uses: actions/cache@v4 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 with: path: | inputs/remote/ @@ -147,7 +147,7 @@ jobs: - name: Restore Zenodo files from cache id: cache-zenodo - uses: actions/cache@v4 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 with: path: | inputs/remote/ @@ -167,7 +167,9 @@ jobs: julia-version: ${{ env.JULIA_VERSION }} - name: Prepend Julia install to PATH - run: echo "${{ steps.setup-env.outputs.julia-bindir }}" >> "$GITHUB_PATH" + env: + JULIA_BINDIR: ${{ steps.setup-env.outputs.julia-bindir }} + run: printf '%s\n' "$JULIA_BINDIR" >> "$GITHUB_PATH" - name: Setup environment variables run: | From 99b101f8c68352ba6d24f6443b3cceaf2d2d334f Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 11:48:43 -0600 Subject: [PATCH 03/13] fix(ci): remove custom CA cert env and rename setup job - drop NODE_EXTRA_CA_CERTS from setup-reeds-env composite action\n- rename CI setup job id from env-prime to reeds-env\n- update downstream needs to reference reeds-env --- .github/actions/setup-reeds-env/action.yml | 2 -- .github/workflows/python-app.yaml | 8 ++++---- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/actions/setup-reeds-env/action.yml b/.github/actions/setup-reeds-env/action.yml index d514763d..3227292e 100644 --- a/.github/actions/setup-reeds-env/action.yml +++ b/.github/actions/setup-reeds-env/action.yml @@ -63,8 +63,6 @@ runs: uses: julia-actions/setup-julia@5c9647d97b78a5debe5164e9eec09d653d29bd71 # v2 with: version: ${{ inputs.julia-version }} - env: - NODE_EXTRA_CA_CERTS: /usr/local/share/ca-certificates/nrel_ca_chain.crt - name: Instantiate Julia project if: ${{ fromJSON(inputs.setup-julia) && fromJSON(inputs.instantiate-julia) }} diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index e58a3af6..5519d686 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -26,8 +26,8 @@ env: JULIA_VERSION: '1.12.1' jobs: - env-prime: - name: Prime Python and Julia caches + reeds-env: + name: ReEDS Python and Julia env runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -45,7 +45,7 @@ jobs: zenodo-setup: name: "Get Zenodo files" needs: - - env-prime + - reeds-env runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -94,7 +94,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 needs: - - env-prime + - reeds-env - zenodo-setup permissions: contents: read From a7221fa78e3abd8bf8925649b0773205e2ae2e0b Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 11:57:21 -0600 Subject: [PATCH 04/13] fix(ci): satisfy zizmor pedantic findings - extract R2X translation logic to scripts/run_r2x.py (uv --script compatible)\n- call run_r2x.py from r2x-check via uvx\n- scope workflow/job permissions to least privilege and document write grants\n- add explicit CI job names for pedantic readability checks --- .github/actions/r2x-check/action.yaml | 40 ++++---------------- .github/workflows/build-docs.yaml | 5 ++- .github/workflows/labeler.yaml | 6 ++- .github/workflows/python-app.yaml | 8 ++-- scripts/run_r2x.py | 54 +++++++++++++++++++++++++++ 5 files changed, 75 insertions(+), 38 deletions(-) create mode 100644 scripts/run_r2x.py diff --git a/.github/actions/r2x-check/action.yaml b/.github/actions/r2x-check/action.yaml index b3025c34..66f6cb7f 100644 --- a/.github/actions/r2x-check/action.yaml +++ b/.github/actions/r2x-check/action.yaml @@ -49,42 +49,18 @@ runs: - name: Run R2X translation shell: bash -le {0} env: - R2X_RUN_PATH: ${{ inputs.reeds-run-path }} + R2X_REEDS_RUN_PATH: ${{ inputs.reeds-run-path }} + R2X_SCENARIO: ${{ steps.resolve.outputs.scenario }} R2X_SOLVE_YEAR: ${{ inputs.solve-year }} R2X_WEATHER_YEAR: ${{ inputs.weather-year }} - R2X_SCENARIO: ${{ steps.resolve.outputs.scenario }} R2X_SYSTEM_JSON: ${{ steps.resolve.outputs.system_json }} run: | - uv run --with "r2x-reeds>=0.3.5" python - <<'PY' - import os - from pathlib import Path - - from loguru import logger - from r2x_core import DataStore, PluginContext - from r2x_core.logger import setup_logging - from r2x_reeds import ReEDSConfig, ReEDSParser - - run_path = Path(os.environ["R2X_RUN_PATH"]).expanduser().resolve() - solve_year = int(os.environ["R2X_SOLVE_YEAR"]) - weather_year = int(os.environ["R2X_WEATHER_YEAR"]) - scenario = os.environ["R2X_SCENARIO"] - system_json = os.environ["R2X_SYSTEM_JSON"] - - setup_logging() - - config = ReEDSConfig( - weather_year=weather_year, - solve_year=solve_year, - scenario=scenario, - ) - - store = DataStore.from_plugin_config(config, path=run_path) - ctx = PluginContext(config=config, store=store) - parser = ReEDSParser.from_context(ctx) - result_ctx = parser.run() - result_ctx.system.to_json(system_json) - logger.info("Built R2X system for scenario '{}' at '{}'", scenario, system_json) - PY + uvx --from "r2x-reeds>=0.3.5" python scripts/run_r2x.py \ + --reeds-run-path "$R2X_REEDS_RUN_PATH" \ + --scenario "$R2X_SCENARIO" \ + --solve-year "$R2X_SOLVE_YEAR" \ + --weather-year "$R2X_WEATHER_YEAR" \ + --system-json "$R2X_SYSTEM_JSON" - name: Save R2X system uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index d1722783..cf1dec16 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -17,11 +17,14 @@ on: - main permissions: - contents: write + contents: read jobs: build: + name: build and publish docs runs-on: ubuntu-latest + permissions: + contents: write # Required for peaceiris/actions-gh-pages publish to gh-pages. steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml index 324fb6d0..a1bcfcbf 100644 --- a/.github/workflows/labeler.yaml +++ b/.github/workflows/labeler.yaml @@ -10,12 +10,14 @@ concurrency: permissions: contents: read - pull-requests: write - issues: write jobs: labeler: name: apply labels + permissions: + contents: read + pull-requests: write # Required to add/remove pull-request labels. + issues: write # Required because PR labels are issue labels in the API. if: github.event.pull_request.draft == false timeout-minutes: 5 runs-on: ubuntu-latest diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 5519d686..9daf527c 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -32,7 +32,7 @@ jobs: timeout-minutes: 15 permissions: contents: read - actions: write + actions: write # Required to save/update micromamba cache. steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -50,7 +50,7 @@ jobs: timeout-minutes: 15 permissions: contents: read - actions: write + actions: write # Required to save/update Zenodo data cache. steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: @@ -91,6 +91,7 @@ jobs: run: python .github/scripts/download_test_zenodo_files.py run-ReEDS: + name: run ReEDS model matrix runs-on: ubuntu-latest timeout-minutes: 60 needs: @@ -98,7 +99,7 @@ jobs: - zenodo-setup permissions: contents: read - actions: write + actions: write # Required to save/update LFS and Zenodo caches. strategy: fail-fast: false max-parallel: 4 @@ -222,6 +223,7 @@ jobs: ${{ env.RUN_FOLDER }}/outputs/**/*.h5 run-R2X: + name: run R2X compatibility matrix runs-on: ubuntu-latest timeout-minutes: 30 needs: diff --git a/scripts/run_r2x.py b/scripts/run_r2x.py new file mode 100644 index 00000000..0d215acd --- /dev/null +++ b/scripts/run_r2x.py @@ -0,0 +1,54 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "r2x-reeds>=0.3.5", +# ] +# /// + +from __future__ import annotations + +import argparse +from pathlib import Path + +from loguru import logger +from r2x_core import DataStore, PluginContext +from r2x_core.logger import setup_logging +from r2x_reeds import ReEDSConfig, ReEDSParser + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Run R2X translation for a ReEDS case.") + parser.add_argument("--reeds-run-path", required=True, type=Path) + parser.add_argument("--scenario", required=True) + parser.add_argument("--solve-year", required=True, type=int) + parser.add_argument("--weather-year", required=True, type=int) + parser.add_argument("--system-json", required=True, type=Path) + return parser.parse_args() + + +def main() -> None: + args = parse_args() + + run_path = args.reeds_run_path.expanduser().resolve() + + setup_logging() + + config = ReEDSConfig( + weather_year=args.weather_year, + solve_year=args.solve_year, + scenario=args.scenario, + ) + + store = DataStore.from_plugin_config(config, path=run_path) + ctx = PluginContext(config=config, store=store) + parser = ReEDSParser.from_context(ctx) + result_ctx = parser.run() + + system_json = args.system_json.expanduser().resolve() + result_ctx.system.to_json(system_json) + logger.info("Built R2X system for scenario '{}' at '{}'", args.scenario, system_json) + + +if __name__ == "__main__": + main() From 43badef56b910955739675a88f2b314ed80821dd Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 12:03:00 -0600 Subject: [PATCH 05/13] refactor(ci): inline R2X check and enforce pedantic zizmor - remove local r2x-check composite action indirection\n- run scripts/run_r2x.py directly in python-app workflow via uvx\n- keep artifact upload behavior in CI job\n- run zizmor with --pedantic by default in workflow-quality --- .github/actions/r2x-check/action.yaml | 71 ------------------------- .github/workflows/python-app.yaml | 37 ++++++++++--- .github/workflows/workflow-quality.yaml | 2 +- 3 files changed, 32 insertions(+), 78 deletions(-) delete mode 100644 .github/actions/r2x-check/action.yaml diff --git a/.github/actions/r2x-check/action.yaml b/.github/actions/r2x-check/action.yaml deleted file mode 100644 index 66f6cb7f..00000000 --- a/.github/actions/r2x-check/action.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: r2x-compatibility -description: "Action to run R2X." -inputs: - reeds-run-path: - required: true - scenario: - python-version: - default: "3.11" - weather-year: - default: "2012" - solve-year: - default: "2035" - -outputs: - scenario: - description: Resolved scenario name used for export. - value: ${{ steps.resolve.outputs.scenario }} - system-json: - description: Filename of the exported R2X system JSON. - value: ${{ steps.resolve.outputs.system_json }} - -runs: - using: composite - steps: - - name: Resolve scenario name - id: resolve - shell: bash -le {0} - env: - INPUT_SCENARIO: ${{ inputs.scenario }} - INPUT_RUN_PATH: ${{ inputs.reeds-run-path }} - run: | - scenario="${INPUT_SCENARIO}" - if [ -z "$scenario" ]; then - scenario="$(basename "$INPUT_RUN_PATH")" - fi - system_json="${scenario}_system.json" - echo "scenario=$scenario" >> "$GITHUB_OUTPUT" - echo "system_json=$system_json" >> "$GITHUB_OUTPUT" - - - name: Install uv and set the python version - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - python-version: ${{ inputs.python-version }} - enable-cache: true - restore-cache: true - save-cache: true - ignore-nothing-to-cache: true - - - name: Run R2X translation - shell: bash -le {0} - env: - R2X_REEDS_RUN_PATH: ${{ inputs.reeds-run-path }} - R2X_SCENARIO: ${{ steps.resolve.outputs.scenario }} - R2X_SOLVE_YEAR: ${{ inputs.solve-year }} - R2X_WEATHER_YEAR: ${{ inputs.weather-year }} - R2X_SYSTEM_JSON: ${{ steps.resolve.outputs.system_json }} - run: | - uvx --from "r2x-reeds>=0.3.5" python scripts/run_r2x.py \ - --reeds-run-path "$R2X_REEDS_RUN_PATH" \ - --scenario "$R2X_SCENARIO" \ - --solve-year "$R2X_SOLVE_YEAR" \ - --weather-year "$R2X_WEATHER_YEAR" \ - --system-json "$R2X_SYSTEM_JSON" - - - name: Save R2X system - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - with: - name: r2x-${{ steps.resolve.outputs.scenario }} - compression-level: 9 - retention-days: 1 - path: ${{ steps.resolve.outputs.system_json }} diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 9daf527c..ffe19a73 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -251,10 +251,35 @@ jobs: with: name: ${{ matrix.scenario }} path: ${{ format('{0}-{1}', matrix.scenario, matrix.solve-year) }} - - name: R2X compatibility - uses: ./.github/actions/r2x-check - id: r2x-check + + - name: Install uv and set Python version + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + python-version: "3.11" + enable-cache: true + restore-cache: true + save-cache: true + ignore-nothing-to-cache: true + + - name: Run R2X compatibility + env: + R2X_REEDS_RUN_PATH: ${{ format('{0}-{1}', matrix.scenario, matrix.solve-year) }} + R2X_SCENARIO: ${{ matrix.scenario }} + R2X_SOLVE_YEAR: ${{ matrix.solve-year }} + R2X_WEATHER_YEAR: "2012" + R2X_SYSTEM_JSON: ${{ format('{0}_system.json', matrix.scenario) }} + run: | + uvx --from "r2x-reeds>=0.3.5" python scripts/run_r2x.py \ + --reeds-run-path "$R2X_REEDS_RUN_PATH" \ + --scenario "$R2X_SCENARIO" \ + --solve-year "$R2X_SOLVE_YEAR" \ + --weather-year "$R2X_WEATHER_YEAR" \ + --system-json "$R2X_SYSTEM_JSON" + + - name: Save R2X system + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: - reeds-run-path: ${{ format('{0}-{1}', matrix.scenario, matrix.solve-year) }} - scenario: ${{ matrix.scenario }} - solve-year: ${{ matrix.solve-year }} + name: r2x-${{ matrix.scenario }} + compression-level: 9 + retention-days: 1 + path: ${{ format('{0}_system.json', matrix.scenario) }} diff --git a/.github/workflows/workflow-quality.yaml b/.github/workflows/workflow-quality.yaml index b71d455f..a7d8e7a8 100644 --- a/.github/workflows/workflow-quality.yaml +++ b/.github/workflows/workflow-quality.yaml @@ -33,7 +33,7 @@ jobs: version: "0.9.4" - name: Run zizmor - run: uvx zizmor .github + run: uvx zizmor --pedantic .github actionlint: name: actionlint From f1e79b49758fbde9a6d1f8f9da85b97156168481 Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 13:28:28 -0600 Subject: [PATCH 06/13] fix(ci): restore docs required check name Set build-docs job name back to 'build' so existing branch protection check context is reported. --- .github/workflows/build-docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-docs.yaml b/.github/workflows/build-docs.yaml index cf1dec16..24f72ed6 100644 --- a/.github/workflows/build-docs.yaml +++ b/.github/workflows/build-docs.yaml @@ -21,7 +21,7 @@ permissions: jobs: build: - name: build and publish docs + name: build runs-on: ubuntu-latest permissions: contents: write # Required for peaceiris/actions-gh-pages publish to gh-pages. From eed7aeb784f0a2e161424bf8a3e11ff51e76a92a Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 14:06:10 -0600 Subject: [PATCH 07/13] ci: pin actionlint installer and fix labels --- .github/dependabot.yml | 8 ++++---- .github/labeler.yaml | 4 +--- .github/workflows/workflow-quality.yaml | 5 ++++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c4a05ef9..f48eaf06 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,7 +9,7 @@ updates: default-days: 7 open-pull-requests-limit: 2 labels: - - "dependenabot" + - "dependencies" reviewers: - "kennedy-mindermann" commit-message: @@ -25,7 +25,7 @@ updates: default-days: 7 open-pull-requests-limit: 2 labels: - - "dependenabot" + - "dependencies" reviewers: - "kennedy-mindermann" commit-message: @@ -41,7 +41,7 @@ updates: default-days: 7 open-pull-requests-limit: 2 labels: - - "dependenabot" + - "dependencies" reviewers: - "kennedy-mindermann" commit-message: @@ -56,7 +56,7 @@ updates: cooldown: default-days: 7 labels: - - "dependenabot" + - "dependencies" reviewers: - "kennedy-mindermann" commit-message: diff --git a/.github/labeler.yaml b/.github/labeler.yaml index 198cf14d..d8c6d3d4 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -1,8 +1,6 @@ -dependenabot: +dependencies: - head-branch: - '^dependabot/.+' - -dependencies: - changed-files: - any-glob-to-any-file: - "environment.yml" diff --git a/.github/workflows/workflow-quality.yaml b/.github/workflows/workflow-quality.yaml index a7d8e7a8..d35249e9 100644 --- a/.github/workflows/workflow-quality.yaml +++ b/.github/workflows/workflow-quality.yaml @@ -46,5 +46,8 @@ jobs: - name: Run actionlint run: | - bash <(curl -s https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash) + ACTIONLINT_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/rhysd/actionlint/e7d448ef7507c20fc4c88a95d0c448b848cd6127/scripts/download-actionlint.bash" + curl -fsSL "$ACTIONLINT_INSTALL_SCRIPT_URL" -o download-actionlint.bash + echo "28a0e78b3230372051c5a77840125aa2c8dc7804fce3696ef29dd52001ec3a8f download-actionlint.bash" | sha256sum -c - + bash download-actionlint.bash ./actionlint From 0e020761ad040f2f97765e7adf57bf71f5c714fc Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 15:25:18 -0600 Subject: [PATCH 08/13] ci: gate heavy runs and upload failure diagnostics --- .github/workflows/python-app.yaml | 45 +++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index ffe19a73..db75bdd3 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -5,11 +5,22 @@ on: pull_request: branches: - main - paths-ignore: - - 'docs/**' - - '*.md' - - 'LICENSE' - - '.gitignore' + paths: + - ".github/actions/setup-reeds-env/**" + - ".github/scripts/download_test_zenodo_files.py" + - ".github/workflows/python-app.yaml" + - "environment.yml" + - "Project.toml" + - "Manifest.toml" + - "runbatch.py" + - "d_solve*.py" + - "createmodel.gms" + - "reeds/**" + - "ReEDS_Augur/**" + - "Augur.py" + - "inputs/**" + - "tests/**" + - "scripts/run_r2x.py" concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -186,12 +197,15 @@ jobs: run: | echo "$GAMSLICE_CONTENT" > "${MAMBA_ENV_PATH}/lib/python3.11/site-packages/gamspy_base/gamslice.txt" + - name: Set run folder + env: + SCENARIO: ${{ matrix.scenario }} + run: echo "RUN_FOLDER=$GITHUB_WORKSPACE/runs/${batch}_${SCENARIO}" >> "$GITHUB_ENV" + - name: Run ReEDS model env: SCENARIO: ${{ matrix.scenario }} - run: | - python runbatch.py -b "$batch" -c test -s "$SCENARIO" - echo "RUN_FOLDER=$GITHUB_WORKSPACE/runs/${batch}_${SCENARIO}" >> "$GITHUB_ENV" + run: python runbatch.py -b "$batch" -c test -s "$SCENARIO" - name: Print GAMS log if: runner.debug == '1' @@ -209,6 +223,21 @@ jobs: SCENARIO: ${{ matrix.scenario }} run: python -m pytest tests/test_outputs.py --casepath "$GITHUB_WORKSPACE/runs/${batch}_${SCENARIO}" + - name: Upload ReEDS failure diagnostics + if: ${{ failure() }} + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: ${{ format('{0}-failure-diagnostics', matrix.scenario) }} + compression-level: 9 + retention-days: 7 + if-no-files-found: ignore + path: | + ${{ env.RUN_FOLDER }}/gamslog.txt + ${{ env.RUN_FOLDER }}/meta.csv + ${{ env.RUN_FOLDER }}/outputs/**/*.log + ${{ env.RUN_FOLDER }}/outputs/**/*.err + ${{ env.RUN_FOLDER }}/outputs/**/*.lst + - name: Save ReEDS run uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 with: From 10a7390ecd72e38469b2764f5906c1df14c53d6c Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 15:41:45 -0600 Subject: [PATCH 09/13] ci: collect targeted failure diagnostics artifacts --- .github/workflows/python-app.yaml | 76 ++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index db75bdd3..ac9cf76d 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -223,6 +223,75 @@ jobs: SCENARIO: ${{ matrix.scenario }} run: python -m pytest tests/test_outputs.py --casepath "$GITHUB_WORKSPACE/runs/${batch}_${SCENARIO}" + - name: Collect ReEDS failure diagnostics + if: ${{ failure() }} + env: + CASE_NAME: ${{ matrix.scenario }} + run: | + DIAG_ROOT="$RUNNER_TEMP/reeds-failure-diagnostics/$CASE_NAME" + mkdir -p "$DIAG_ROOT" + MANIFEST="$DIAG_ROOT/manifest.txt" + + copy_if_exists() { + local src="$1" + local rel="$2" + if [[ -f "$src" ]]; then + mkdir -p "$(dirname "$DIAG_ROOT/$rel")" + cp "$src" "$DIAG_ROOT/$rel" + printf 'included: %s\n' "$src" >> "$MANIFEST" + else + printf 'missing: %s\n' "$src" >> "$MANIFEST" + fi + } + + latest_by_year_iteration() { + local search_dir="$1" + local regex="$2" + local best_file="" + local best_year=-1 + local best_iter=-1 + + [[ -d "$search_dir" ]] || return 0 + + while IFS= read -r file; do + local base year iter year_num iter_num + base="$(basename "$file")" + if [[ "$base" =~ $regex ]]; then + year="${BASH_REMATCH[1]}" + iter="${BASH_REMATCH[2]}" + year_num=$((10#$year)) + iter_num=$((10#$iter)) + if (( year_num > best_year || (year_num == best_year && iter_num > best_iter) )); then + best_year=$year_num + best_iter=$iter_num + best_file="$file" + fi + fi + done < <(find "$search_dir" -maxdepth 1 -type f) + + printf '%s\n' "$best_file" + } + + copy_if_exists "$RUN_FOLDER/gamslog.txt" "gamslog.txt" + copy_if_exists "$RUN_FOLDER/meta.csv" "meta.csv" + copy_if_exists "$RUN_FOLDER/lstfiles/1_Inputs.lst" "lstfiles/1_Inputs.lst" + + latest_case_lst="$(latest_by_year_iteration "$RUN_FOLDER/lstfiles" "^${CASE_NAME}_([0-9]+)i([0-9]+)\\.lst$")" + if [[ -n "$latest_case_lst" ]]; then + copy_if_exists "$latest_case_lst" "lstfiles/$(basename "$latest_case_lst")" + else + printf 'missing: %s\n' "$RUN_FOLDER/lstfiles/${CASE_NAME}_{year}i{iteration}.lst" >> "$MANIFEST" + fi + + copy_if_exists "$RUN_FOLDER/lstfiles/report_${CASE_NAME}.lst" "lstfiles/report_${CASE_NAME}.lst" + + latest_pras_log="$(latest_by_year_iteration "$RUN_FOLDER/ReEDS_Augur/PRAS" "^PRAS_([0-9]+)i([0-9]+)\\.log$")" + if [[ -n "$latest_pras_log" ]]; then + copy_if_exists "$latest_pras_log" "ReEDS_Augur/PRAS/$(basename "$latest_pras_log")" + else + printf 'missing: %s\n' "$RUN_FOLDER/ReEDS_Augur/PRAS/PRAS_{year}i{iteration}.log" >> "$MANIFEST" + fi + - name: Upload ReEDS failure diagnostics if: ${{ failure() }} uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 @@ -231,12 +300,7 @@ jobs: compression-level: 9 retention-days: 7 if-no-files-found: ignore - path: | - ${{ env.RUN_FOLDER }}/gamslog.txt - ${{ env.RUN_FOLDER }}/meta.csv - ${{ env.RUN_FOLDER }}/outputs/**/*.log - ${{ env.RUN_FOLDER }}/outputs/**/*.err - ${{ env.RUN_FOLDER }}/outputs/**/*.lst + path: ${{ format('{0}/reeds-failure-diagnostics/{1}', runner.temp, matrix.scenario) }} - name: Save ReEDS run uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 From 64e9c8e582264b730862e2a733db502941ce8578 Mon Sep 17 00:00:00 2001 From: pesap Date: Thu, 23 Apr 2026 16:01:17 -0600 Subject: [PATCH 10/13] ci: simplify failure diagnostics artifact globs --- .github/workflows/python-app.yaml | 77 +++---------------------------- 1 file changed, 7 insertions(+), 70 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index ac9cf76d..4eed5acb 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -223,75 +223,6 @@ jobs: SCENARIO: ${{ matrix.scenario }} run: python -m pytest tests/test_outputs.py --casepath "$GITHUB_WORKSPACE/runs/${batch}_${SCENARIO}" - - name: Collect ReEDS failure diagnostics - if: ${{ failure() }} - env: - CASE_NAME: ${{ matrix.scenario }} - run: | - DIAG_ROOT="$RUNNER_TEMP/reeds-failure-diagnostics/$CASE_NAME" - mkdir -p "$DIAG_ROOT" - MANIFEST="$DIAG_ROOT/manifest.txt" - - copy_if_exists() { - local src="$1" - local rel="$2" - if [[ -f "$src" ]]; then - mkdir -p "$(dirname "$DIAG_ROOT/$rel")" - cp "$src" "$DIAG_ROOT/$rel" - printf 'included: %s\n' "$src" >> "$MANIFEST" - else - printf 'missing: %s\n' "$src" >> "$MANIFEST" - fi - } - - latest_by_year_iteration() { - local search_dir="$1" - local regex="$2" - local best_file="" - local best_year=-1 - local best_iter=-1 - - [[ -d "$search_dir" ]] || return 0 - - while IFS= read -r file; do - local base year iter year_num iter_num - base="$(basename "$file")" - if [[ "$base" =~ $regex ]]; then - year="${BASH_REMATCH[1]}" - iter="${BASH_REMATCH[2]}" - year_num=$((10#$year)) - iter_num=$((10#$iter)) - if (( year_num > best_year || (year_num == best_year && iter_num > best_iter) )); then - best_year=$year_num - best_iter=$iter_num - best_file="$file" - fi - fi - done < <(find "$search_dir" -maxdepth 1 -type f) - - printf '%s\n' "$best_file" - } - - copy_if_exists "$RUN_FOLDER/gamslog.txt" "gamslog.txt" - copy_if_exists "$RUN_FOLDER/meta.csv" "meta.csv" - copy_if_exists "$RUN_FOLDER/lstfiles/1_Inputs.lst" "lstfiles/1_Inputs.lst" - - latest_case_lst="$(latest_by_year_iteration "$RUN_FOLDER/lstfiles" "^${CASE_NAME}_([0-9]+)i([0-9]+)\\.lst$")" - if [[ -n "$latest_case_lst" ]]; then - copy_if_exists "$latest_case_lst" "lstfiles/$(basename "$latest_case_lst")" - else - printf 'missing: %s\n' "$RUN_FOLDER/lstfiles/${CASE_NAME}_{year}i{iteration}.lst" >> "$MANIFEST" - fi - - copy_if_exists "$RUN_FOLDER/lstfiles/report_${CASE_NAME}.lst" "lstfiles/report_${CASE_NAME}.lst" - - latest_pras_log="$(latest_by_year_iteration "$RUN_FOLDER/ReEDS_Augur/PRAS" "^PRAS_([0-9]+)i([0-9]+)\\.log$")" - if [[ -n "$latest_pras_log" ]]; then - copy_if_exists "$latest_pras_log" "ReEDS_Augur/PRAS/$(basename "$latest_pras_log")" - else - printf 'missing: %s\n' "$RUN_FOLDER/ReEDS_Augur/PRAS/PRAS_{year}i{iteration}.log" >> "$MANIFEST" - fi - - name: Upload ReEDS failure diagnostics if: ${{ failure() }} uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 @@ -300,7 +231,13 @@ jobs: compression-level: 9 retention-days: 7 if-no-files-found: ignore - path: ${{ format('{0}/reeds-failure-diagnostics/{1}', runner.temp, matrix.scenario) }} + path: | + ${{ env.RUN_FOLDER }}/gamslog.txt + ${{ env.RUN_FOLDER }}/meta.csv + ${{ env.RUN_FOLDER }}/lstfiles/1_Inputs.lst + ${{ env.RUN_FOLDER }}/lstfiles/${{ matrix.scenario }}_*i*.lst + ${{ env.RUN_FOLDER }}/lstfiles/report_${{ matrix.scenario }}.lst + ${{ env.RUN_FOLDER }}/ReEDS_Augur/PRAS/PRAS_*i*.log - name: Save ReEDS run uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 From b3b7d45725a7986e314c38f89221d5f38b3f2c94 Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 24 Apr 2026 14:33:39 -0600 Subject: [PATCH 11/13] ci: restore pull_request paths-ignore trigger --- .github/workflows/python-app.yaml | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/.github/workflows/python-app.yaml b/.github/workflows/python-app.yaml index 4eed5acb..5f69a91d 100644 --- a/.github/workflows/python-app.yaml +++ b/.github/workflows/python-app.yaml @@ -5,22 +5,11 @@ on: pull_request: branches: - main - paths: - - ".github/actions/setup-reeds-env/**" - - ".github/scripts/download_test_zenodo_files.py" - - ".github/workflows/python-app.yaml" - - "environment.yml" - - "Project.toml" - - "Manifest.toml" - - "runbatch.py" - - "d_solve*.py" - - "createmodel.gms" - - "reeds/**" - - "ReEDS_Augur/**" - - "Augur.py" - - "inputs/**" - - "tests/**" - - "scripts/run_r2x.py" + paths-ignore: + - 'docs/**' + - '*.md' + - 'LICENSE' + - '.gitignore' concurrency: group: ${{ github.workflow }}-${{ github.ref }} From fc98684b64783b69f117b938ca8c3b73441dd9de Mon Sep 17 00:00:00 2001 From: pesap Date: Mon, 27 Apr 2026 20:33:22 -0600 Subject: [PATCH 12/13] fixup! ci: simplify GitHub workflow setup and labeling --- .github/actions/setup-reeds-env/action.yml | 9 --------- .github/labeler.yaml | 10 ++++------ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/.github/actions/setup-reeds-env/action.yml b/.github/actions/setup-reeds-env/action.yml index 3227292e..0818d3c8 100644 --- a/.github/actions/setup-reeds-env/action.yml +++ b/.github/actions/setup-reeds-env/action.yml @@ -18,10 +18,6 @@ inputs: description: Run julia --project=. instantiate.jl. required: false default: "true" - remove-conda-julia: - description: Remove julia from micromamba env before setup-julia. - required: false - default: "true" outputs: mamba-env-path: @@ -44,11 +40,6 @@ runs: post-cleanup: "all" cache-environment-key: ${{ runner.os }}-conda-${{ hashFiles('environment.yml') }} - - name: Remove conda Julia package - if: ${{ fromJSON(inputs.setup-python) && fromJSON(inputs.setup-julia) && fromJSON(inputs.remove-conda-julia) }} - shell: bash -le {0} - run: micromamba remove -y julia || true - - name: Restore Julia cache if: ${{ fromJSON(inputs.setup-julia) }} uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4 diff --git a/.github/labeler.yaml b/.github/labeler.yaml index d8c6d3d4..441c0a34 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -42,10 +42,10 @@ tests: model_changes: - changed-files: - any-glob-to-any-file: + - "*.gms" - "**/*.gms" - "runbatch.py" - "d_solve*.py" - - "createmodel.gms" - "reeds/**" - "ReEDS_Augur/**" - "Augur.py" @@ -60,10 +60,6 @@ data_changes: - "postprocessing/**/inputs/**" - "**/*.csv" - "**/*.h5" - - "**/*.hdf5" - - "**/*.parquet" - - "**/*.xlsx" - - "**/*.xls" - "sources.csv" - "sources_documentation.md" @@ -87,4 +83,6 @@ reeds2pras: inputs: - changed-files: - - any-glob-to-any-file: "inputs/**" + - any-glob-to-any-file: + - "inputs/**" + - "cases.csv" From 798d638f928f4509b0ae5a5fa37ad65957199978 Mon Sep 17 00:00:00 2001 From: pesap Date: Fri, 1 May 2026 12:20:26 -0600 Subject: [PATCH 13/13] chore: Renaming inputs to switches and just leaving cases changes for labels --- .github/labeler.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/labeler.yaml b/.github/labeler.yaml index 441c0a34..e0558a5b 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -81,8 +81,7 @@ reeds2pras: - changed-files: - any-glob-to-any-file: "reeds2pras/**" -inputs: +switches: - changed-files: - any-glob-to-any-file: - - "inputs/**" - "cases.csv"