_docs.yml

name: build docs

on:
  workflow_call:
    inputs:
      build-environment:
        required: true
        type: string
        description: Top-level label for what's being built/tested.
      docker-image:
        required: true
        type: string
        description: Docker image to run in.
      push:
        required: false
        type: boolean
        default: false
        description: If set, push the docs to the docs website.
      run-doxygen:
        required: false
        type: boolean
        default: false
        description: If set, will enable C++ API doc generation using doxygen / breathe / exhale.
      sync-tag:
        required: false
        type: string
        default: ""
        description: |
          If this is set, our linter will use this to make sure that every other
          job with the same `sync-tag` is identical.
      s3-bucket:
        description: S3 bucket to download artifact
        required: false
        type: string
        default: "gha-artifacts"
      aws-role-to-assume:
        description: role to assume for downloading artifacts
        required: false
        type: string
        default: ""
      upload-aws-role-to-assume:
        description: role to assume for downloading artifacts
        required: false
        type: string
        default: ""
      runner_prefix:
        description: prefix for runner label
        type: string
        default: ""
      use-arc:
        required: false
        type: boolean
        default: false
        description: If true, use ARC (OSDC) runner path instead of EC2.
      python-version:
        required: false
        type: string
        default: ""
        description: Python version to use for the OSDC build.
      compiler:
        required: false
        type: string
        default: ""
        description: Compiler to use for the OSDC build.
      cuda-version:
        required: false
        type: string
        default: ""
        description: CUDA version to use for the OSDC build.
    secrets:
      GH_PYTORCHBOT_TOKEN:
        required: false
        description: Permissions for pushing to the docs site.

jobs:
  build-docs:
    # Don't run on forked repos.
    if: github.repository_owner == 'pytorch' && !inputs.use-arc
    runs-on: ${{ matrix.runner }}
    environment: ${{ (github.ref == 'refs/heads/main' || startsWith(github.event.ref, 'refs/tags/v')) && 'pytorchbot-env' || '' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - docs_type: cpp
            # We recently seeing lots of exit code 137 running this in Docker indicating
            # an OOM issue when running the job, so this upgrades the runner from 4xlarge
            # to the next available tier of 12xlarge. So much memory just to generate cpp
            # doc
            runner: ${{ inputs.runner_prefix }}linux.12xlarge.memory
            # TODO: Nightly cpp docs take longer and longer to finish (more than 3h now)
            # Let's try to figure out how this can be improved
            timeout-minutes: 360
          - docs_type: python
            runner: ${{ inputs.runner_prefix }}linux.c7i.2xlarge
            timeout-minutes: 45
    # Set a fixed name for this job instead of using the current matrix-generated name, i.e. build-docs (cpp, linux.12xlarge, 180)
    # The current name requires updating the database last docs push query from test-infra every time the matrix is updated
    name: build-docs-${{ matrix.docs_type }}-${{ inputs.push }}
    steps:
      - name: Setup SSH (Click me for login details)
        uses: pytorch/test-infra/.github/actions/setup-ssh@main
        with:
          github-secret: ${{ secrets.GITHUB_TOKEN }}
          instructions: |
            All builds are done inside the container, to start an interactive session run:
              docker exec -it $(docker container ps --format '{{.ID}}') bash
            To start Python docs build type:
              cd docs && make html && make coverage

      - name: Setup Linux
        uses: pytorch/pytorch/.github/actions/setup-linux@main

      - name: Login to ECR
        uses: ./.github/actions/ecr-login
        with:
          aws-role-to-assume: ${{ inputs.aws-role-to-assume }}

      - name: Calculate docker image
        id: calculate-docker-image
        uses: pytorch/test-infra/.github/actions/calculate-docker-image@main
        with:
          docker-image-name: ${{ inputs.docker-image }}

      - name: Pull docker image
        uses: pytorch/test-infra/.github/actions/pull-docker-image@main
        with:
          docker-image: ${{ steps.calculate-docker-image.outputs.docker-image }}

      - name: Download build artifacts
        uses: ./.github/actions/download-build-artifacts
        with:
          name: ${{ inputs.build-environment }}
          s3-bucket: ${{ inputs.s3-bucket }}

      - name: Generate netrc (only for docs-push)
        if: inputs.push
        env:
          GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
        run: |
          # sometimes .netrc exists as a directory even though this is the temp folder
          rm -rf "${RUNNER_TEMP}/.netrc"
          # set credentials for https pushing
          echo "machine github.com" > "${RUNNER_TEMP}/.netrc"
          echo "login pytorchbot" >> "${RUNNER_TEMP}/.netrc"
          echo "password ${GITHUB_PYTORCHBOT_TOKEN}" >> "${RUNNER_TEMP}/.netrc"

      - name: Build ${{ matrix.docs_type }} docs
        timeout-minutes: ${{ matrix.timeout-minutes }}
        id: build-docs
        env:
          # After https://github.com/pytorch/pytorch/pull/88373, pull workflow can now be run periodically,
          # so using a schedule event to determine if the docs should be pushed or not doesn't hold true
          # anymore
          WITH_PUSH: ${{ inputs.push }}
          DOCKER_IMAGE: ${{ inputs.docker-image }}
          DOCS_TYPE: ${{ matrix.docs_type }}
          RUN_DOXYGEN: ${{ inputs.run-doxygen }}
          BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
        run: |
          set -ex
          # Convert refs/tags/v1.12.0rc3 into 1.12
          if [[ "${GITHUB_REF}" =~ ^refs/tags/v([0-9]+\.[0-9]+)\.* ]]; then
            target="${BASH_REMATCH[1]}"
          else
            target="main"
          fi
          # detached container should get cleaned up by teardown_ec2_linux
          container_name=$(docker run \
            -e BUILD_ENVIRONMENT \
            -e MAX_JOBS="$(nproc --ignore=2)" \
            -e SHA1="$GITHUB_SHA" \
            -e DOCS_VERSION="${target}" \
            -e DOCS_TYPE \
            -e RUN_DOXYGEN \
            -e WITH_PUSH \
            --env-file="/tmp/github_env_${GITHUB_RUN_ID}" \
            -e GITHUB_WORKSPACE=/var/lib/jenkins/workspace \
            --security-opt seccomp=unconfined \
            --cap-add=SYS_PTRACE \
            --tty \
            --detach \
            --user jenkins \
            -v "${RUNNER_TEMP}/.netrc":/var/lib/jenkins/.netrc \
            -v "${GITHUB_WORKSPACE}:/var/lib/jenkins/workspace" \
            -w /var/lib/jenkins/workspace \
            "${DOCKER_IMAGE}"
          )
          docker exec -t "${container_name}" bash -c "sudo chown -R jenkins . && pip install $(echo dist/*.whl)[opt-einsum] && ./.ci/pytorch/${DOCS_TYPE}_doc_push_script.sh"

      - name: Chown workspace
        uses: ./.github/actions/chown-workspace
        if: always()

      - name: configure aws credentials
        if : ${{ inputs.upload-aws-role-to-assume != '' }}
        uses: aws-actions/configure-aws-credentials@ececac1a45f3b08a01d2dd070d28d111c5fe6722 # v4.1.0
        with:
          role-to-assume: ${{ inputs.upload-aws-role-to-assume }}
          role-session-name: gha-linux-test
          aws-region: us-east-1

      - name: Upload Python Docs Preview
        uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
        if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'python' && steps.build-docs.outcome == 'success' }}
        with:
          retention-days: 14
          s3-bucket: doc-previews
          if-no-files-found: error
          path: pytorch_docs/main/
          s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}

      - name: Upload C++ Docs Preview
        uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
        if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
        with:
          retention-days: 14
          if-no-files-found: error
          s3-bucket: doc-previews
          path: cppdocs/
          s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/cppdocs

      - name: Post C++ Docs Coverage Comment
        if: ${{ github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          PR_NUM="${{ github.event.pull_request.number }}"
          MARKER="<!-- cpp-docs-coverage -->"

          # Only post if the PR touches docs/cpp
          if ! gh pr diff "$PR_NUM" --name-only | grep -q '^docs/cpp/'; then
            echo "No changes to docs/cpp/, skipping coverage comment."
            exit 0
          fi

          # Skip if we already posted
          if gh pr view "$PR_NUM" --json comments --jq '.comments[].body' | grep -q "$MARKER"; then
            echo "Coverage comment already posted, skipping."
            exit 0
          fi

          body="${MARKER}
          ## C++ Docs

          - [Doc Preview](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/index.html)
          - [API Coverage Report](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/_coverage/cpp_coverage.txt)
          - [HTML Issues Report](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/_coverage/cpp_html_issues.txt)"

          if [ -f cppdocs/_coverage/index.html ]; then
            body="${body}
          - [Coverxygen Report (interactive)](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/_coverage/index.html)"
          fi

          gh pr comment "$PR_NUM" --body "$body"

      - name: Upload C++ Docs Preview (nightly dry-run)
        if: ${{ !inputs.push && github.event_name != 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
        run: |
          # Use aws s3 cp --recursive for fast parallel uploads instead of uploading files one by one
          aws s3 cp cppdocs/ s3://doc-previews/pytorch/pytorch/nightly-${{ github.sha }}/cppdocs --recursive --quiet
          echo "C++ docs preview available at:"
          echo "https://docs-preview.pytorch.org/pytorch/pytorch/nightly-${{ github.sha }}/cppdocs/index.html"

      - name: Teardown Linux
        uses: pytorch/test-infra/.github/actions/teardown-linux@main
        if: always()

  build-docs-osdc:
    if: github.repository_owner == 'pytorch' && inputs.use-arc
    permissions:
      id-token: write
      contents: read
      actions: read
    runs-on: ${{ matrix.runner }}
    container:
      image: ${{ inputs.docker-image }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - docs_type: cpp
            runner: ${{ inputs.runner_prefix }}l-x86iavx512-48-384
            timeout-minutes: 360
          - docs_type: python
            runner: ${{ inputs.runner_prefix }}l-x86iavx512-16-128
            timeout-minutes: 45
    name: build-docs-${{ matrix.docs_type }}-${{ inputs.push }}
    steps:
      - name: Setup Linux
        id: setup-linux
        uses: pytorch/pytorch/.github/actions/setup-linux@main
        with:
          use-arc: true
          python-version: ${{ inputs.python-version }}
          compiler: ${{ inputs.compiler }}
          cuda-version: ${{ inputs.cuda-version }}
          submodules: false
          github-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Configure AWS credentials
        id: aws-creds
        continue-on-error: true
        uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7
        with:
          role-to-assume: arn:aws:iam::308535385114:role/arc
          aws-region: us-east-1
          role-duration-seconds: 18000

      - name: Download build artifacts
        uses: pytorch/pytorch/.github/actions/download-build-artifacts@main
        with:
          name: ${{ inputs.build-environment }}
          s3-bucket: ${{ inputs.s3-bucket }}
          use-gha: ${{ steps.aws-creds.outcome != 'success' }}

      - name: Generate netrc (only for docs-push)
        if: inputs.push
        env:
          GITHUB_PYTORCHBOT_TOKEN: ${{ secrets.GH_PYTORCHBOT_TOKEN }}
        run: |
          rm -rf "${HOME}/.netrc"
          echo "machine github.com" > "${HOME}/.netrc"
          echo "login pytorchbot" >> "${HOME}/.netrc"
          echo "password ${GITHUB_PYTORCHBOT_TOKEN}" >> "${HOME}/.netrc"

      - name: Build ${{ matrix.docs_type }} docs
        timeout-minutes: ${{ matrix.timeout-minutes }}
        id: build-docs
        env:
          WITH_PUSH: ${{ inputs.push }}
          DOCS_TYPE: ${{ matrix.docs_type }}
          RUN_DOXYGEN: ${{ inputs.run-doxygen }}
          BUILD_ENVIRONMENT: ${{ inputs.build-environment }}
          SHA1: ${{ github.sha }}
        shell: bash
        run: |
          set -ex
          if [[ "${GITHUB_REF}" =~ ^refs/tags/v([0-9]+\.[0-9]+)\.* ]]; then
            target="${BASH_REMATCH[1]}"
          else
            target="main"
          fi
          export DOCS_VERSION="${target}"
          pip install $(echo dist/*.whl)[opt-einsum]
          ./.ci/pytorch/${DOCS_TYPE}_doc_push_script.sh

      - name: Upload Python Docs Preview
        uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
        if: ${{ steps.aws-creds.outcome == 'success' && github.event_name == 'pull_request' && matrix.docs_type == 'python' && steps.build-docs.outcome == 'success' }}
        with:
          retention-days: 14
          s3-bucket: doc-previews
          if-no-files-found: error
          path: pytorch_docs/main/
          s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}

      - name: Upload C++ Docs Preview
        uses: seemethere/upload-artifact-s3@baba72d0712b404f646cebe0730933554ebce96a # v5.1.0
        if: ${{ steps.aws-creds.outcome == 'success' && github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
        with:
          retention-days: 14
          if-no-files-found: error
          s3-bucket: doc-previews
          path: cppdocs/
          s3-prefix: pytorch/pytorch/${{ github.event.pull_request.number }}/cppdocs

      - name: Post C++ Docs Coverage Comment
        if: ${{ steps.aws-creds.outcome == 'success' && github.event_name == 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          PR_NUM="${{ github.event.pull_request.number }}"
          MARKER="<!-- cpp-docs-coverage -->"

          # Only post if the PR touches docs/cpp
          if ! gh pr diff "$PR_NUM" --name-only | grep -q '^docs/cpp/'; then
            echo "No changes to docs/cpp/, skipping coverage comment."
            exit 0
          fi

          # Skip if we already posted
          if gh pr view "$PR_NUM" --json comments --jq '.comments[].body' | grep -q "$MARKER"; then
            echo "Coverage comment already posted, skipping."
            exit 0
          fi

          body="${MARKER}
          ## C++ Docs

          - [Doc Preview](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/index.html)
          - [API Coverage Report](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/_coverage/cpp_coverage.txt)
          - [HTML Issues Report](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/_coverage/cpp_html_issues.txt)"

          if [ -f cppdocs/_coverage/index.html ]; then
            body="${body}
          - [Coverxygen Report (interactive)](https://docs-preview.pytorch.org/pytorch/pytorch/${PR_NUM}/cppdocs/_coverage/index.html)"
          fi

          gh pr comment "$PR_NUM" --body "$body"

      - name: Upload C++ Docs Preview (nightly dry-run)
        if: ${{ steps.aws-creds.outcome == 'success' && !inputs.push && github.event_name != 'pull_request' && matrix.docs_type == 'cpp' && steps.build-docs.outcome == 'success' }}
        run: |
          # Unlike EC2 runners, the OSDC container doesn't have aws CLI pre-installed
          pip install awscli==1.29.40
          aws s3 cp cppdocs/ s3://doc-previews/pytorch/pytorch/nightly-${{ github.sha }}/cppdocs --recursive --quiet
          echo "C++ docs preview available at:"
          echo "https://docs-preview.pytorch.org/pytorch/pytorch/nightly-${{ github.sha }}/cppdocs/index.html"

  memory-viz-tests:
    # Tests for torch memory visualizer
    # tag shangdiy for any issue with this test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: '20'
      - name: Run MemoryViz JS tests (nonretryable)
        run: node test/profiler/test_memory_viz.js