From 995891e5c645a811bfbf6831506972b23353289d Mon Sep 17 00:00:00 2001
From: cpelley <carwyn.pelley@metoffice.gov.uk>
Date: Thu, 9 Apr 2026 12:56:57 +0100
Subject: [PATCH 1/2] test hugo build

---
 .github/workflows/hugo.yml | 123 +++++++++++++++++++++++++++++++++++++
 1 file changed, 123 insertions(+)
 create mode 100644 .github/workflows/hugo.yml

diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml
new file mode 100644
index 0000000..560c3ea
--- /dev/null
+++ b/.github/workflows/hugo.yml
@@ -0,0 +1,123 @@
+# This workflow builds the Hugo site and deploys it to GitHub Pages.
+# see https://gohugo.io/host-and-deploy/host-on-github-pages/
+name: Build and deploy
+on:
+  push:
+    branches:
+      - main
+      - pull-requests
+
+  workflow_dispatch:
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+concurrency:
+  group: pages
+  cancel-in-progress: false
+defaults:
+  run:
+    shell: bash
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      DART_SASS_VERSION: 1.99.0
+      GO_VERSION: 1.26.1
+      HUGO_VERSION: 0.160.0
+      NODE_VERSION: 24.14.1
+      TZ: Europe/Oslo
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      # === CUSTOM DAGRUNNER ===
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+
+      - name: Build reference documentation
+        run: |
+          ./docs/gen_docs dagrunner ./docs
+      # ======
+
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v6
+      - name: Create directory for user-specific executable files
+        run: |
+          mkdir -p "${HOME}/.local"
+      - name: Install Dart Sass
+        run: |
+          curl -sLJO "https://github.com/sass/dart-sass/releases/download/${DART_SASS_VERSION}/dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz"
+          tar -C "${HOME}/.local" -xf "dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz"
+          rm "dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz"
+          echo "${HOME}/.local/dart-sass" >> "${GITHUB_PATH}"
+      - name: Install Hugo
+        run: |
+          curl -sLJO "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz"
+          mkdir "${HOME}/.local/hugo"
+          tar -C "${HOME}/.local/hugo" -xf "hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz"
+          rm "hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz"
+          echo "${HOME}/.local/hugo" >> "${GITHUB_PATH}"
+      - name: Verify installations
+        run: |
+          echo "Dart Sass: $(sass --version)"
+          echo "Go: $(go version)"
+          echo "Hugo: $(hugo version)"
+          echo "Node.js: $(node --version)"
+      - name: Install Node.js dependencies
+        run: |
+          [[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true
+      - name: Configure Git
+        run: |
+          git config core.quotepath false
+      - name: Cache restore
+        id: cache-restore
+        uses: actions/cache/restore@v5
+        with:
+          path: ${{ runner.temp }}/hugo_cache
+          key: hugo-${{ github.run_id }}
+          restore-keys:
+            hugo-
+      - name: Build the site
+        run: |
+          cd docs/
+          hugo build \
+            --gc \
+            --minify \
+            --baseURL "${{ steps.pages.outputs.base_url }}/" \
+            --cacheDir "${{ runner.temp }}/hugo_cache"
+      - name: Cache save
+        id: cache-save
+        uses: actions/cache/save@v5
+        with:
+          path: ${{ runner.temp }}/hugo_cache
+          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./public
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v5
\ No newline at end of file

From 1e3b93ede52120510b53a278126f14bd50a5c207 Mon Sep 17 00:00:00 2001
From: cpelley <carwyn.pelley@metoffice.gov.uk>
Date: Thu, 9 Apr 2026 12:59:31 +0100
Subject: [PATCH 2/2] debug

---
 .github/workflows/hugo.yml  | 123 -----------------------------
 .github/workflows/main.yml  | 151 ++++++++++++++++++++++++++++++++++++
 .github/workflows/tests.yml |  23 ++----
 3 files changed, 157 insertions(+), 140 deletions(-)
 delete mode 100644 .github/workflows/hugo.yml
 create mode 100644 .github/workflows/main.yml

diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml
deleted file mode 100644
index 560c3ea..0000000
--- a/.github/workflows/hugo.yml
+++ /dev/null
@@ -1,123 +0,0 @@
-# This workflow builds the Hugo site and deploys it to GitHub Pages.
-# see https://gohugo.io/host-and-deploy/host-on-github-pages/
-name: Build and deploy
-on:
-  push:
-    branches:
-      - main
-      - pull-requests
-
-  workflow_dispatch:
-permissions:
-  contents: read
-  pages: write
-  id-token: write
-concurrency:
-  group: pages
-  cancel-in-progress: false
-defaults:
-  run:
-    shell: bash
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    env:
-      DART_SASS_VERSION: 1.99.0
-      GO_VERSION: 1.26.1
-      HUGO_VERSION: 0.160.0
-      NODE_VERSION: 24.14.1
-      TZ: Europe/Oslo
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          submodules: recursive
-          fetch-depth: 0
-
-      # === CUSTOM DAGRUNNER ===
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: 3.x
-
-      - name: Build reference documentation
-        run: |
-          ./docs/gen_docs dagrunner ./docs
-      # ======
-
-      - name: Setup Go
-        uses: actions/setup-go@v6
-        with:
-          go-version: ${{ env.GO_VERSION }}
-          cache: false
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: ${{ env.NODE_VERSION }}
-      - name: Setup Pages
-        id: pages
-        uses: actions/configure-pages@v6
-      - name: Create directory for user-specific executable files
-        run: |
-          mkdir -p "${HOME}/.local"
-      - name: Install Dart Sass
-        run: |
-          curl -sLJO "https://github.com/sass/dart-sass/releases/download/${DART_SASS_VERSION}/dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz"
-          tar -C "${HOME}/.local" -xf "dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz"
-          rm "dart-sass-${DART_SASS_VERSION}-linux-x64.tar.gz"
-          echo "${HOME}/.local/dart-sass" >> "${GITHUB_PATH}"
-      - name: Install Hugo
-        run: |
-          curl -sLJO "https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz"
-          mkdir "${HOME}/.local/hugo"
-          tar -C "${HOME}/.local/hugo" -xf "hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz"
-          rm "hugo_extended_${HUGO_VERSION}_linux-amd64.tar.gz"
-          echo "${HOME}/.local/hugo" >> "${GITHUB_PATH}"
-      - name: Verify installations
-        run: |
-          echo "Dart Sass: $(sass --version)"
-          echo "Go: $(go version)"
-          echo "Hugo: $(hugo version)"
-          echo "Node.js: $(node --version)"
-      - name: Install Node.js dependencies
-        run: |
-          [[ -f package-lock.json || -f npm-shrinkwrap.json ]] && npm ci || true
-      - name: Configure Git
-        run: |
-          git config core.quotepath false
-      - name: Cache restore
-        id: cache-restore
-        uses: actions/cache/restore@v5
-        with:
-          path: ${{ runner.temp }}/hugo_cache
-          key: hugo-${{ github.run_id }}
-          restore-keys:
-            hugo-
-      - name: Build the site
-        run: |
-          cd docs/
-          hugo build \
-            --gc \
-            --minify \
-            --baseURL "${{ steps.pages.outputs.base_url }}/" \
-            --cacheDir "${{ runner.temp }}/hugo_cache"
-      - name: Cache save
-        id: cache-save
-        uses: actions/cache/save@v5
-        with:
-          path: ${{ runner.temp }}/hugo_cache
-          key: ${{ steps.cache-restore.outputs.cache-primary-key }}
-      - name: Upload artifact
-        uses: actions/upload-pages-artifact@v3
-        with:
-          path: ./public
-  deploy:
-    environment:
-      name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - name: Deploy to GitHub Pages
-        id: deployment
-        uses: actions/deploy-pages@v5
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
new file mode 100644
index 0000000..bd62bfb
--- /dev/null
+++ b/.github/workflows/main.yml
@@ -0,0 +1,151 @@
+name: Main Update
+
+on:
+  push:
+    branches:
+      - main  # Triggers when commits are pushed or merged into main
+    tags:
+      - '[0-9]*'  # Triggers when a tag starting with number is pushed
+  pull_request:  # uncomment for testing purposes
+
+jobs:
+  coverage_report_doc_update:
+    runs-on: ubuntu-latest
+    env:
+      FORCE_UPDATE_DOCS: true  # Set env var to force docs update (for testing purposes)
+      FORCE_UPDATE_COVERAGE: true  # Set env var to force coverage report update (for testing purposes)
+
+    steps:
+      # SETUP
+      ############################
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Get reference name (branch or tag)
+        id: get-ref-name
+        run: |
+          echo "ref_name=main" | tee -a $GITHUB_ENV
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+          cache: 'pip'
+
+      - name: Add checkout directory to PYTHONPATH
+        run: echo "PYTHONPATH=$(pwd):$PYTHONPATH" | tee -a $GITHUB_ENV
+
+      - name: Check if tests need running (coverage report cache hit)
+        uses: actions/cache/restore@v5
+        id: report-cache-restore
+        with:
+          path: |
+            coverage_output.txt
+          key: coverage_report_${{ hashFiles('pyproject.toml', 'dagrunner/**/*.py') }}
+
+      - name: Check if docs need building (docs cache hit)
+        uses: actions/cache/restore@v5
+        id: docs-cache-restore
+        with:
+          lookup-only: true
+          path: |
+            docs/
+          key: docs_cache_${{ hashFiles('pyproject.toml', 'dagrunner/**/*.py', 'docs/**') }}
+
+      - name: Install dependencies
+        if: steps.report-cache-restore.outputs.cache-hit != 'true' || steps.docs-cache-restore.outputs.cache-hit != 'true'
+        id: install-dependencies
+        run: |
+          pip install .[tests,visualisation,dev]
+          pip uninstall dagrunner -y
+
+      - name: Set up SSH for localhost
+        if: steps.report-cache-restore.outputs.cache-hit != 'true'
+        run: |
+          ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa
+          cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+          ssh-keyscan -H $(hostname) >> ~/.ssh/known_hosts
+          chmod 600 ~/.ssh/authorized_keys
+
+      # TESTS (inc. generate and publish coverage report)
+      ############################
+      - name: Run pytest + generate coverage report
+        if: steps.report-cache-restore.outputs.cache-hit != 'true'
+        run: pytest --cov=dagrunner --cov-report=term --cov-report=html | tee coverage_output.txt; test ${PIPESTATUS[0]} -eq 0
+
+      - name: Save cached coverage report
+        uses: actions/cache/save@v5
+        id: report-cache-save
+        with:
+          path: |
+            coverage_output.txt
+          key: ${{ steps.report-cache-restore.outputs.cache-primary-key }}
+
+      - name: Checkout reference coverage report
+        if: steps.report-cache-restore.outputs.cache-hit != 'true'
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+          path: covref
+          ref: coverage-reference
+
+      - name: Commit and push coverage report
+        if: steps.report-cache-restore.outputs.cache-hit != 'true'
+        run: |
+          cd covref
+          ref_name=${{ steps.get-ref-name.outputs.ref_name }}
+
+          git config user.name github-actions[bot]
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          rm -rf ${ref_name}
+          mkdir ${ref_name}
+
+          cp ../coverage_output.txt ${ref_name}/.
+          cp -r ../htmlcov/ ${ref_name}/.
+          git add -f ${ref_name}/
+
+          echo "=== git status after add ==="
+          git status -s
+
+          git commit -a --amend --no-edit || echo "No changes to commit"
+          git push origin -f coverage-reference
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      # DOCUMENTATION
+      ############################
+      - name: Build documentation
+        if: steps.docs-cache-restore.outputs.cache-hit != 'true'
+        run: |
+          ./docs/gen_docs dagrunner ./docs
+
+      - name: Checkout gh-pages branch
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 1
+          path: gh-pages
+          ref: gh-pages
+
+      - name: Update gh-pages branch with new documentation
+        if: steps.docs-cache-restore.outputs.cache-hit != 'true'
+        run: |
+          cd gh-pages/
+          ref_name=${{ steps.get-ref-name.outputs.ref_name }}
+
+          git config user.name github-actions[bot]
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          rm -rf ${ref_name}
+          mkdir ${ref_name}
+
+          cp -r ../docs/ ${ref_name}/.
+          git add -f ${ref_name}/
+
+          echo "=== git status after add ==="
+          git status -s
+
+          git commit -a --amend --no-edit || echo "No changes to commit"
+          git push origin -f gh-pages
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 9825bcf..6b28df6 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -1,6 +1,10 @@
 name: Pull Request Tests
 
-on: [pull_request]
+on:
+  push:
+    branches:
+      - main  # Triggers when commits are pushed or merged into main
+# on: [pull_request]
 
 
 jobs:
@@ -55,7 +59,6 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          ref: ${{ github.head_ref }}  # Ensure branch is checked out, not detached state (so we can push a commit later)
           token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Set up Python
@@ -87,25 +90,11 @@ jobs:
 
       # TESTS ('main' branch)
       ############################
-      - name: Cache ref branch coverage report
+      - name: Get coverage report for main
         id: cache-ref-coverage
         uses: actions/cache@v4
         with:
           path: ref/coverage_output.txt
-          key: ref-${{ github.event.pull_request.base.sha }}
-
-      - name: Checkout ref branch
-        if: steps.cache-ref-coverage.outputs.cache-hit != 'true'
-        uses: actions/checkout@v4
-        with:
-          path: ref
-          ref: ${{ github.base_ref }}
-
-      - name: Run tests with coverage for ref branch
-        if: steps.cache-ref-coverage.outputs.cache-hit != 'true'
-        run: |
-          cd ref
-          pytest --maxfail=0 --continue-on-collection-errors --cov=dagrunner --cov-report=term --cov-report=html | tee coverage_output.txt || true
 
       # TESTS (compare coverage)
       ############################