Fix gh-pages docs generation

.github/docker/Dockerfile-cd-docs-local

@@ -0,0 +1,24 @@
+FROM ubuntu:24.04
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY .github/docker/cd_docs_apt_packages.txt /tmp/cd_docs_apt_packages.txt
+
+RUN apt-get update --fix-missing && \
+    xargs -a /tmp/cd_docs_apt_packages.txt apt-get install -y --no-install-recommends && \
+    rm -rf /var/lib/apt/lists/* /tmp/cd_docs_apt_packages.txt
+
+WORKDIR /workspace
+
+COPY requirements.txt requirements_doc.txt requirements_dev.txt ./
+
+RUN python3 -m venv /opt/venv
+ENV PATH="/opt/venv/bin:${PATH}"
+
+RUN pip install --upgrade pip && \
+    pip install -r requirements.txt -r requirements_doc.txt -r requirements_dev.txt
+
+COPY .github/docker/run_cd_docs_ci_local_in_container.sh /usr/local/bin/run_cd_docs_ci_local_in_container.sh
+RUN chmod +x /usr/local/bin/run_cd_docs_ci_local_in_container.sh
+
+ENTRYPOINT ["/usr/local/bin/run_cd_docs_ci_local_in_container.sh"]

.github/docker/cd_docs_apt_packages.txt

@@ -0,0 +1,19 @@
+ca-certificates
+cmake
+doxygen
+g++
+gcc
+git
+graphviz
+libhdf5-dev
+libxml2-dev
+libxslt-dev
+make
+python3
+python3-dev
+python3-numpy
+python3-pip
+python3-venv
+python3-yaml
+rsync
+swig

.github/docker/run_cd_docs_ci_local_in_container.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BUILD_DIR="tmp"
+
+cd /workspace
+
+if [[ -n "${CI_SOURCE_REF:-}" ]]; then
+  echo "== Source ref requested: ${CI_SOURCE_REF} =="
+fi
+if [[ -n "${CI_SOURCE_SHA:-}" ]]; then
+  echo "== Expected source SHA: ${CI_SOURCE_SHA} =="
+fi
+if git rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+  ACTUAL_SHA="$(git rev-parse HEAD)"
+  echo "== Container source SHA: ${ACTUAL_SHA} =="
+  if [[ -n "${CI_SOURCE_SHA:-}" ]] && [[ "${ACTUAL_SHA}" != "${CI_SOURCE_SHA}" ]]; then
+    echo "ERROR: Container source SHA does not match expected SHA."
+    exit 1
+  fi
+fi
+
+echo "== Tool versions =="
+python --version
+swig -version | head -n 1 || true
+cmake --version | head -n 1
+doxygen --version
+dot -V
+
+echo "== Build and verify docs (single source of truth) =="
+if [[ ! -x tools/cd_docs_build_and_verify.sh ]]; then
+  chmod +x tools/cd_docs_build_and_verify.sh
+fi
+BUILD_DIR="${BUILD_DIR}" tools/cd_docs_build_and_verify.sh
+
+echo "== SUCCESS: Local cd_docs reproduction completed =="

.github/workflows/cd_docs.yml

@@ -5,6 +5,7 @@ on:
   push:
     branches:
       - master
+      - ghpagesfail
   release:
     types:
       - published
@@ -32,26 +33,18 @@ jobs:
         uses: actions/checkout@v6
         with:
           fetch-depth: 0
+          clean: true
 
       - name: Setup Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.13.5"
+          # Keep docs on a stable interpreter for current Sphinx/AutoAPI stack.
+          python-version: "3.12"
 
       - name: Install system dependencies
         run: |
           sudo apt-get update --fix-missing
-          sudo apt-get install \
-            libxml2-dev \
-            libxslt-dev \
-            libhdf5-dev \
-            doxygen \
-            graphviz \
-            python3 \
-            python3-pip \
-            python3-dev \
-            python3-numpy \
-            python3-yaml
+          xargs -a .github/docker/cd_docs_apt_packages.txt sudo apt-get install -y
           #python3 -m pip install psycopg2-binary==2.9.5
 
       - name: Install Python dependencies
@@ -67,74 +60,10 @@ jobs:
           doxygen --version
           dot -V
 
-      - name: Run CMAKE
+      - name: Build and verify docs (single source of truth)
         run: |
-          rm -rf "${BUILD_DIR}"
-          mkdir -p "${BUILD_DIR}"
-          cd "${BUILD_DIR}"
-          Python3_ROOT=$(python3 -c 'import sys; print(sys.exec_prefix)') \
-            CFLAGS='-Wno-missing-field-initializers' \
-            cmake .. -DFORCE_EXAMPLES=ON -DWITH_DOC=ON
-
-      - name: Run MakeFile
-        run: make
-        working-directory: ${{ env.BUILD_DIR }}
-
-      - name: Verify Python AutoAPI output
-        run: |
-          set -euo pipefail
-          mapfile -t DOCS_CANDIDATES < <(find . -type f -path '*/doc/html/index.html' -not -path './gh-pages/*' | sed 's#/index.html$##' | sort -u)
-          if [[ "${#DOCS_CANDIDATES[@]}" -eq 0 ]]; then
-            echo "ERROR: Could not locate Sphinx HTML output directory (*/doc/html)."
-            echo "---- Candidate doc directories ----"
-            find . -maxdepth 4 -type d -name doc | sort || true
-            echo "---- Candidate html directories ----"
-            find . -maxdepth 6 -type d -name html | sort || true
-            exit 1
-          fi
-
-          DOCS_HTML_DIR=""
-          AUTOAPI_DIR=""
-          BEST_AUTOAPI_INDEX_COUNT=-1
-          for CANDIDATE_DIR in "${DOCS_CANDIDATES[@]}"; do
-            CANDIDATE_AUTOAPI_DIR="${CANDIDATE_DIR}/autoapi"
-            CANDIDATE_AUTOAPI_INDEX_COUNT=$(find "${CANDIDATE_AUTOAPI_DIR}" -type f -name index.html 2>/dev/null | wc -l)
-            echo "Candidate docs HTML directory: ${CANDIDATE_DIR} (AutoAPI index pages: ${CANDIDATE_AUTOAPI_INDEX_COUNT})"
-            if [[ "${CANDIDATE_AUTOAPI_INDEX_COUNT}" -gt "${BEST_AUTOAPI_INDEX_COUNT}" ]]; then
-              BEST_AUTOAPI_INDEX_COUNT="${CANDIDATE_AUTOAPI_INDEX_COUNT}"
-              DOCS_HTML_DIR="${CANDIDATE_DIR}"
-              AUTOAPI_DIR="${CANDIDATE_AUTOAPI_DIR}"
-            fi
-          done
-
-          echo "Using docs HTML directory: ${DOCS_HTML_DIR}"
-          echo "DOCS_HTML_DIR=${DOCS_HTML_DIR}" >> "$GITHUB_ENV"
-
-          if [[ "${BEST_AUTOAPI_INDEX_COUNT}" -lt 1 ]]; then
-            echo "ERROR: AutoAPI output was not found in any docs HTML directory."
-            echo "---- All AutoAPI directories under workspace ----"
-            find . -type d -path '*/autoapi' | sort || true
-            echo "---- All AutoAPI index files under workspace ----"
-            find . -type f -path '*/autoapi/*/index.html' | sort | head -n 200 || true
-            echo "---- Sphinx config excerpt ----"
-            find . -type f -path '*/doc/_build/conf.py' -print -exec sed -n '1,260p' {} \; | grep -E 'autoapi_(dirs|ignore|file_patterns|python_use_implicit_namespaces)' || true
-            echo "---- docs HTML listing ----"
-            ls -la "${DOCS_HTML_DIR}" || true
-            exit 1
-          fi
-
-          echo "Using AutoAPI directory: ${AUTOAPI_DIR}"
-          AUTOAPI_INDEX_COUNT="${BEST_AUTOAPI_INDEX_COUNT}"
-          echo "Detected ${AUTOAPI_INDEX_COUNT} AutoAPI index page(s)."
-
-          if [[ ! -f "${DOCS_HTML_DIR}/api.html" ]] || ! grep -Eq 'autoapi/.*/index.html' "${DOCS_HTML_DIR}/api.html"; then
-            echo "ERROR: API landing page does not link to Python AutoAPI pages."
-            echo "---- api.html excerpt ----"
-            sed -n '460,620p' "${DOCS_HTML_DIR}/api.html" || true
-            exit 1
-          fi
-
-          find "${AUTOAPI_DIR}" -maxdepth 3 -type f -name index.html | sort | head -n 20
+          chmod +x tools/cd_docs_build_and_verify.sh
+          BUILD_DIR="${BUILD_DIR}" tools/cd_docs_build_and_verify.sh
 
       - name: Determine deployment target
         run: |

README.md

@@ -33,6 +33,31 @@ Documentation
 -------------
 The official documentation for DLite can be found on https://sintef.github.io/dlite/.
 
+Reproduce docs CI locally (Docker)
+----------------------------------
+To reproduce the `cd_docs.yml` docs build and AutoAPI checks locally in a
+Docker container, run:
+
+```shell
+./tools/run_cd_docs_ci_local.sh
+```
+
+To run against the exact checked-in commit (matching `actions/checkout` source
+semantics), provide a git ref explicitly:
+
+```shell
+./tools/run_cd_docs_ci_local.sh HEAD
+```
+
+This runs the same core steps as the GitHub docs workflow (configure, build,
+and AutoAPI verification) and prints the same diagnostics used in CI.
+Both GitHub Actions and the local Docker runner call the shared script
+`tools/cd_docs_build_and_verify.sh`, so docs build/check logic is maintained
+in one place.
+System package parity is also centralized in
+`.github/docker/cd_docs_apt_packages.txt`, which is used by both the workflow
+and the local Docker image.
+
 
 Installation
 ------------

doc/conf.py.in

@@ -7,6 +7,7 @@ import os
 import sys
 from pathlib import Path
 import re
+import shutil
 from typing import TYPE_CHECKING
 from subprocess import call
 
@@ -77,25 +78,32 @@ if plugin_root.exists():
         if plugin_dir.is_dir()
     ]
 
-# Prefer the built Python package tree (stable module namespace), but fall
-# back to source tree if docs are built without package generation.
+# Stage API sources into the docs build tree and point AutoAPI there.
+# This avoids CI path/readability issues when mixing source and build trees.
 autoapi_build_dir = Path("@CMAKE_BINARY_DIR@/bindings/python/dlite")
 autoapi_source_dir = Path("@CMAKE_SOURCE_DIR@/bindings/python")
+autoapi_stage_root = Path(__file__).resolve().parent / "_autoapi_sources"
+autoapi_stage_pkg = autoapi_stage_root / "dlite"
 
-def _has_python_sources(path: Path) -> bool:
-    if not path.exists() or not path.is_dir():
-        return False
-    return any(path.rglob("*.py")) or any(path.rglob("*.pyi"))
-
-# Prefer the source tree for deterministic docs in CI. The build tree can
-# exist but still be incomplete depending on build order/configuration.
-autoapi_dirs = []
-if _has_python_sources(autoapi_source_dir):
-    autoapi_dirs.append(str(autoapi_source_dir))
-if _has_python_sources(autoapi_build_dir):
-    autoapi_dirs.append(str(autoapi_build_dir))
-if not autoapi_dirs:
-    autoapi_dirs = [str(autoapi_source_dir)]
+if autoapi_stage_root.exists():
+    shutil.rmtree(autoapi_stage_root)
+autoapi_stage_pkg.mkdir(parents=True, exist_ok=True)
+
+for source_file in sorted(autoapi_source_dir.glob("*.py")):
+    if source_file.name == "doxy2swig.py":
+        continue
+    shutil.copy2(source_file, autoapi_stage_pkg / source_file.name)
+
+# Keep generated package entry points when available.
+for generated_name in ("__init__.py", "paths.py"):
+    generated_path = autoapi_build_dir / generated_name
+    if generated_path.exists():
+        shutil.copy2(generated_path, autoapi_stage_pkg / generated_name)
+
+if not (autoapi_stage_pkg / "__init__.py").exists():
+    (autoapi_stage_pkg / "__init__.py").write_text('"""DLite Python API."""\n', encoding="utf-8")
+
+autoapi_dirs = [str(autoapi_stage_root)]
 
 print(f"[conf.py] AutoAPI dirs: {autoapi_dirs}")
 autoapi_type = "python"

tools/cd_docs_build_and_verify.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Single source of truth for docs CI build + AutoAPI verification.
+BUILD_DIR="${BUILD_DIR:-tmp}"
+
+echo "== Configure (cmake) =="
+rm -rf "${BUILD_DIR}"
+mkdir -p "${BUILD_DIR}"
+# Provide a writable home for cmake/python cache when running as an arbitrary UID
+mkdir -p "${BUILD_DIR}/.home"
+cd "${BUILD_DIR}"
+Python3_ROOT="$(python3 -c 'import sys; print(sys.exec_prefix)')" \
+  CFLAGS='-Wno-missing-field-initializers' \
+  cmake .. -DFORCE_EXAMPLES=ON -DWITH_DOC=ON
+
+echo "== Build (make) =="
+make VERBOSE=1 2>&1 | tee make-docs.log
+
+cd ..
+
+echo "== Verify Python AutoAPI output =="
+mapfile -t DOCS_CANDIDATES < <(find . -type f -path '*/doc/html/index.html' -not -path './gh-pages/*' | sed 's#/index.html$##' | sort -u)
+if [[ "${#DOCS_CANDIDATES[@]}" -eq 0 ]]; then
+  echo "ERROR: Could not locate Sphinx HTML output directory (*/doc/html)."
+  echo "---- Candidate doc directories ----"
+  find . -maxdepth 4 -type d -name doc | sort || true
+  echo "---- Candidate html directories ----"
+  find . -maxdepth 6 -type d -name html | sort || true
+  exit 1
+fi
+
+DOCS_HTML_DIR=""
+AUTOAPI_DIR=""
+BEST_AUTOAPI_INDEX_COUNT=-1
+for CANDIDATE_DIR in "${DOCS_CANDIDATES[@]}"; do
+  CANDIDATE_AUTOAPI_DIR="${CANDIDATE_DIR}/autoapi"
+  CANDIDATE_AUTOAPI_INDEX_COUNT=$(find "${CANDIDATE_AUTOAPI_DIR}" -type f -name index.html 2>/dev/null | wc -l)
+  echo "Candidate docs HTML directory: ${CANDIDATE_DIR} (AutoAPI index pages: ${CANDIDATE_AUTOAPI_INDEX_COUNT})"
+  if [[ "${CANDIDATE_AUTOAPI_INDEX_COUNT}" -gt "${BEST_AUTOAPI_INDEX_COUNT}" ]]; then
+    BEST_AUTOAPI_INDEX_COUNT="${CANDIDATE_AUTOAPI_INDEX_COUNT}"
+    DOCS_HTML_DIR="${CANDIDATE_DIR}"
+    AUTOAPI_DIR="${CANDIDATE_AUTOAPI_DIR}"
+  fi
+done
+
+echo "Using docs HTML directory: ${DOCS_HTML_DIR}"
+if [[ -n "${GITHUB_ENV:-}" ]]; then
+  echo "DOCS_HTML_DIR=${DOCS_HTML_DIR}" >> "${GITHUB_ENV}"
+fi
+
+if [[ "${BEST_AUTOAPI_INDEX_COUNT}" -lt 1 ]]; then
+  echo "ERROR: AutoAPI output was not found in any docs HTML directory."
+  echo "---- Make/Sphinx log tail ----"
+  if [[ -f "${BUILD_DIR}/make-docs.log" ]]; then
+    tail -n 300 "${BUILD_DIR}/make-docs.log" || true
+  else
+    echo "No ${BUILD_DIR}/make-docs.log found."
+  fi
+  echo "---- All AutoAPI directories under workspace ----"
+  find . -type d -path '*/autoapi' | sort || true
+  echo "---- All AutoAPI index files under workspace ----"
+  find . -type f -path '*/autoapi/*/index.html' | sort | head -n 200 || true
+  echo "---- Sphinx config excerpt ----"
+  find . -type f -path '*/doc/_build/conf.py' -print -exec sed -n '1,260p' {} \; | grep -E 'autoapi_(dirs|ignore|file_patterns|python_use_implicit_namespaces)' || true
+  echo "---- docs HTML listing ----"
+  ls -la "${DOCS_HTML_DIR}" || true
+  exit 1
+fi
+
+echo "Using AutoAPI directory: ${AUTOAPI_DIR}"
+AUTOAPI_INDEX_COUNT="${BEST_AUTOAPI_INDEX_COUNT}"
+echo "Detected ${AUTOAPI_INDEX_COUNT} AutoAPI index page(s)."
+
+if [[ ! -f "${DOCS_HTML_DIR}/api.html" ]] || ! grep -Eq 'autoapi/.*/index.html' "${DOCS_HTML_DIR}/api.html"; then
+  echo "ERROR: API landing page does not link to Python AutoAPI pages."
+  echo "---- api.html excerpt ----"
+  sed -n '460,620p' "${DOCS_HTML_DIR}/api.html" || true
+  exit 1
+fi
+
+find "${AUTOAPI_DIR}" -maxdepth 3 -type f -name index.html | sort | head -n 20
+
+echo "== SUCCESS: docs build + AutoAPI verification completed =="