Regenerate models #6
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Keeps the generated API models in sync with the published OpenAPI specification: every night it regenerates them | |
| # and opens a pull request when the result differs from master. | |
| # | |
| # Two invariants make it safe to run unattended: | |
| # 1. Generation always happens on master, so the output follows the current spec and the current codegen tooling, | |
| # never whatever an older auto-update branch carries. | |
| # 2. The auto-update branch is rebuilt from master instead of appended to, so the diff is always "current spec vs | |
| # current master" and can't resurrect a stale generated file. | |
| name: Regenerate models | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: "0 2 * * *" | |
| concurrency: | |
| group: regenerate-models | |
| cancel-in-progress: false | |
| # Writes go through the service account token below, not through `GITHUB_TOKEN`. | |
| permissions: | |
| contents: read | |
| env: | |
| PYTHON_VERSION: 3.14 | |
| BRANCH_NAME: ci/regenerate-models | |
| # Valid Conventional Commits, so the pull request is mergeable as-is; reviewers retitle it to `fix:`/`feat:` when | |
| # the diff is user-facing. | |
| PR_TITLE: "chore: Regenerate models from the published OpenAPI spec" | |
| ASSIGNEE: vdusek | |
| LABEL: t-tooling | |
| jobs: | |
| regenerate-models: | |
| name: Regenerate models | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout master | |
| uses: actions/checkout@v7 | |
| with: | |
| ref: master | |
| token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| fetch-depth: 0 | |
| - name: Set up uv package manager | |
| uses: astral-sh/setup-uv@v9.0.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: uv run poe install-dev | |
| # Read before regeneration overwrites it, so the pull request can say whether the specification moved. | |
| - name: Read the recorded specification version | |
| id: previous-spec | |
| run: echo "version=$(uv run python -m scripts.openapi_spec recorded-version)" >> "$GITHUB_OUTPUT" | |
| # Downloads the specification, generates from it, and records its version in `pyproject.toml`. | |
| - name: Regenerate models | |
| run: uv run poe generate-models | |
| # Gate on the generated models, not on the specification version: that stamp moves on rebuilds regardless of | |
| # client impact, and doesn't reliably move when the content does (see the pull request body step below). | |
| # Compared against HEAD rather than the index, so nothing staged earlier in the job can hide a change. | |
| - name: Check for model changes | |
| id: changes | |
| run: | | |
| if git diff --quiet HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py; then | |
| echo "Models are already up to date with the published specification." | |
| echo "has-changes=false" >> "$GITHUB_OUTPUT" | |
| else | |
| git diff --stat HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py | |
| echo "has-changes=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| # A previous run may already have these exact models up for review; leave it alone instead of churning an open | |
| # pull request. Only the generated files are compared - the branch being behind master says nothing about | |
| # whether the models on it are still the right ones. | |
| - name: Check whether the models are already up for review | |
| id: review | |
| if: steps.changes.outputs.has-changes == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| run: | | |
| # `--exit-code` reports 2 for "no such branch"; any other failure must not read as an absent branch, or a | |
| # transient network error would retire a current pull request. | |
| git ls-remote --exit-code --heads origin "$BRANCH_NAME" >/dev/null 2>&1 && status=0 || status=$? | |
| if [[ "$status" == "2" ]]; then | |
| echo "No auto-update branch exists yet." | |
| echo "is-new=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| elif [[ "$status" != "0" ]]; then | |
| echo "Failed to look up branch $BRANCH_NAME on the remote (git ls-remote exited $status)." >&2 | |
| exit 1 | |
| fi | |
| # The branch only counts as proposed while a pull request is open on it: a leftover branch - a run that | |
| # failed before opening one, or a review that closed it without deleting the branch - must be rebuilt, or | |
| # its contents would look proposed and silently suppress every future regeneration. An empty result counts | |
| # only once the query itself succeeded, otherwise an API error would replace a live pull request. | |
| if ! PR_NUMBER=$(gh pr list --head "$BRANCH_NAME" --base master --state open --json number --jq '.[0].number // empty'); then | |
| echo "Failed to list open pull requests for $BRANCH_NAME." >&2 | |
| exit 1 | |
| fi | |
| if [[ -z "$PR_NUMBER" ]]; then | |
| echo "Branch $BRANCH_NAME has no open pull request - rebuilding it." | |
| echo "is-new=true" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| git fetch origin "$BRANCH_NAME" | |
| if git diff --quiet FETCH_HEAD -- src/apify_client/_models.py src/apify_client/_typeddicts.py src/apify_client/_literals.py; then | |
| echo "The open pull request already carries these models - nothing to do." | |
| echo "is-new=false" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "is-new=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| # Retire the previous branch before recreating it. Deleting it first also avoids the window where the branch | |
| # tip would equal master, which GitHub reads as an empty pull request and auto-closes. | |
| - name: Retire the superseded pull request | |
| if: steps.changes.outputs.has-changes == 'true' && steps.review.outputs.is-new == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| PR_NUMBER=$(gh pr list --head "$BRANCH_NAME" --base master --state open --json number --jq '.[0].number // empty') | |
| if [[ -n "$PR_NUMBER" ]]; then | |
| gh pr close "$PR_NUMBER" --delete-branch \ | |
| --comment "Superseded by a newer regeneration run: ${RUN_URL}." | |
| echo "Closed superseded PR #${PR_NUMBER}." | |
| elif git ls-remote --exit-code --heads origin "$BRANCH_NAME" >/dev/null 2>&1; then | |
| git push origin --delete "$BRANCH_NAME" | |
| echo "Deleted leftover branch $BRANCH_NAME (no open pull request)." | |
| fi | |
| # Creates the branch at the current master commit and lands the regenerated files as one signed ("Verified") | |
| # commit, via GitHub's createCommitOnBranch mutation. | |
| - name: Commit the regenerated models | |
| id: commit | |
| if: steps.changes.outputs.has-changes == 'true' && steps.review.outputs.is-new == 'true' | |
| uses: apify/actions/signed-commit@v1.4.0 | |
| with: | |
| message: ${{ env.PR_TITLE }} | |
| add: >- | |
| pyproject.toml | |
| src/apify_client/_models.py | |
| src/apify_client/_typeddicts.py | |
| src/apify_client/_literals.py | |
| github-token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| branch: ${{ env.BRANCH_NAME }} | |
| create-branch: "true" | |
| - name: Create the pull request | |
| if: steps.commit.outputs.committed == 'true' | |
| env: | |
| GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/workflows/on_schedule_regenerate_models.yaml | |
| PREVIOUS_SPEC_VERSION: ${{ steps.previous-spec.outputs.version }} | |
| run: | | |
| SPEC_VERSION=$(uv run python -m scripts.openapi_spec recorded-version) | |
| # A moved stamp proves the specification changed; an unchanged one proves nothing, because apify-docs bumps | |
| # `components/version.yaml` in a follow-up `[skip ci]` commit and a deploy can publish new content under | |
| # the old stamp. Say which case this is instead of letting the reviewer read it as a content identity. | |
| if [[ "$SPEC_VERSION" == "$PREVIOUS_SPEC_VERSION" ]]; then | |
| SPEC_LINE="- Specification version: \`${SPEC_VERSION}\` - **unchanged**, which doesn't rule out a specification change: the published stamp can lag its content by a deploy. Only the diff tells." | |
| else | |
| SPEC_LINE="- Specification version: \`${PREVIOUS_SPEC_VERSION}\` -> \`${SPEC_VERSION}\`." | |
| fi | |
| BODY=$(printf '%s\n' \ | |
| "- Regenerates the Pydantic models, TypedDicts, and literal aliases from the [published OpenAPI specification](https://docs.apify.com/api/openapi.json), and records its version in \`pyproject.toml\`." \ | |
| "${SPEC_LINE}" \ | |
| "" \ | |
| "> [!IMPORTANT]" \ | |
| "> Retitle this pull request to \`fix:\` or \`feat:\` when the diff is user-facing, so that it lands in the changelog and triggers a release - \`chore:\` does neither." \ | |
| "" \ | |
| "> Generated by the [Regenerate models](${WORKFLOW_URL}) workflow.") | |
| gh pr create \ | |
| --title "$PR_TITLE" \ | |
| --body "$BODY" \ | |
| --base master \ | |
| --head "$BRANCH_NAME" \ | |
| --assignee "$ASSIGNEE" \ | |
| --label "$LABEL" | |
| # Without this a broken sync stops regeneration silently: GitHub only notifies whoever last touched the cron, and | |
| # no release waits on this workflow. Skipped on manual dispatch so ad-hoc triggers don't spam the channel. | |
| notify_on_failure: | |
| name: Notify Slack on failure | |
| needs: regenerate-models | |
| if: failure() && github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| steps: | |
| - name: Build Slack payload | |
| env: | |
| REPO: ${{ github.repository }} | |
| WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| HEADING: ":red_circle: Nightly model regeneration failed" | |
| run: | | |
| jq -n \ | |
| --arg repo "${REPO}" \ | |
| --arg url "${WORKFLOW_URL}" \ | |
| --arg heading "${HEADING}" \ | |
| '{ | |
| text: "\($heading) in \($repo)", | |
| blocks: [ | |
| { | |
| type: "header", | |
| text: { type: "plain_text", text: $heading, emoji: true } | |
| }, | |
| { | |
| type: "section", | |
| fields: [ | |
| { type: "mrkdwn", text: "*Repository:*\n\($repo)" }, | |
| { type: "mrkdwn", text: "*Workflow run:*\n<\($url)|View on GitHub>" } | |
| ] | |
| }, | |
| { | |
| type: "section", | |
| text: { type: "mrkdwn", text: "The generated API models are no longer being kept in sync with the published OpenAPI specification." } | |
| } | |
| ] | |
| }' > slack-payload.json | |
| - name: Send Slack notification | |
| uses: slackapi/slack-github-action@v4.0.0 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| webhook-type: incoming-webhook | |
| payload-file-path: slack-payload.json |