From b5b3b0592cd10b47b5e2bd4baf0da08b56262a58 Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Wed, 19 Aug 2026 20:39:21 +0100 Subject: [PATCH 1/3] ci: stop apt from hanging Linux jobs until the 6h limit Linux jobs intermittently stalled in "Install Linux dependencies" and ran until the job timeout. The hosted runner mirrorlist points apt at azure.archive.ubuntu.com, which is regularly unreachable; apt falls back to archive.ubuntu.com and can trickle bytes indefinitely. Nothing failed the step: it had no timeout-minutes and apt had no acquire timeout. Two runs were stuck this way at once, on different Node versions (26 and 24), which rules out anything version specific. - add timeout-minutes: 20 to the three Linux setup steps - run apt under `timeout` with explicit Acquire timeouts and retries, and retry `apt-get update` up to three times - drop gcc-10/g++-10 from the test workflow: ~48 MB out of noble universe via the same flaky mirror, and binding.gyp only pins that toolchain when it finds it on PATH, so the default gcc covers the -std=c++20 it asks for. prebuild.yml keeps the pin for ABI compatibility of published binaries. Co-Authored-By: Claude Opus 5 --- .github/workflows/prebuild.yml | 28 ++++++++++++++++++++++++---- .github/workflows/test.yml | 21 +++++++++++++++++---- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/.github/workflows/prebuild.yml b/.github/workflows/prebuild.yml index 74cbc9b9..0f2a809c 100644 --- a/.github/workflows/prebuild.yml +++ b/.github/workflows/prebuild.yml @@ -44,6 +44,7 @@ jobs: # Linux specific setup - name: Install Linux dependencies + timeout-minutes: 20 if: runner.os == 'Linux' run: | # The GitHub-hosted Ubuntu runner image ships a pre-configured @@ -61,8 +62,17 @@ jobs: curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list # Install dependencies - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev gcc-10 g++-10 + # Hosted runners point apt at an azure.archive.ubuntu.com mirror that is + # frequently unreachable. apt then falls back to archive.ubuntu.com, which + # can trickle bytes indefinitely - with no timeout the step never fails, it + # just runs until the 6 hour job limit kills the whole job. + APT_OPTS="-o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=3" + for attempt in 1 2 3; do + sudo timeout 300 apt-get update $APT_OPTS && break + echo "apt-get update stalled or failed (attempt $attempt/3), retrying" + sleep 15 + done + sudo timeout 900 env ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y $APT_OPTS msodbcsql18 mssql-tools18 unixodbc-dev gcc-10 g++-10 # Set compiler echo "CC=gcc-10" >> $GITHUB_ENV @@ -173,14 +183,24 @@ jobs: # Linux specific setup (same as above) - name: Install Linux dependencies + timeout-minutes: 20 if: runner.os == 'Linux' run: | # See note in matching step above for why this rm is necessary. sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev gcc-10 g++-10 + # Hosted runners point apt at an azure.archive.ubuntu.com mirror that is + # frequently unreachable. apt then falls back to archive.ubuntu.com, which + # can trickle bytes indefinitely - with no timeout the step never fails, it + # just runs until the 6 hour job limit kills the whole job. + APT_OPTS="-o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=3" + for attempt in 1 2 3; do + sudo timeout 300 apt-get update $APT_OPTS && break + echo "apt-get update stalled or failed (attempt $attempt/3), retrying" + sleep 15 + done + sudo timeout 900 env ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y $APT_OPTS msodbcsql18 mssql-tools18 unixodbc-dev gcc-10 g++-10 echo "CC=gcc-10" >> $GITHUB_ENV echo "CXX=g++-10" >> $GITHUB_ENV diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ed8c191..de9ea7c1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,13 +30,26 @@ jobs: node-version: ${{ matrix.node }} - name: Install Linux dependencies + timeout-minutes: 20 run: | curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 mssql-tools18 unixodbc-dev gcc-10 g++-10 - echo "CC=gcc-10" >> $GITHUB_ENV - echo "CXX=g++-10" >> $GITHUB_ENV + # Hosted runners point apt at an azure.archive.ubuntu.com mirror that is + # frequently unreachable. apt then falls back to archive.ubuntu.com, which + # can trickle bytes indefinitely - with no timeout the step never fails, it + # just runs until the 6 hour job limit kills the whole job. + APT_OPTS="-o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=3" + for attempt in 1 2 3; do + sudo timeout 300 apt-get update $APT_OPTS && break + echo "apt-get update stalled or failed (attempt $attempt/3), retrying" + sleep 15 + done + # No gcc-10/g++-10 here on purpose: they pull ~48 MB out of noble universe + # via the same flaky mirror, and the default toolchain handles the -std=c++20 + # binding.gyp asks for. binding.gyp only pins gcc-10 when it finds it on PATH, + # so leaving it out is all that is needed. prebuild.yml keeps the pin for ABI + # compatibility of the published binaries. + sudo timeout 900 env ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y $APT_OPTS msodbcsql18 mssql-tools18 unixodbc-dev - name: Run SQL Server uses: potatoqualitee/mssqlsuite@v1.7 From 0c5cb4b4adb7b1e43cdd43494a7df1992a7381a4 Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Wed, 19 Aug 2026 20:46:48 +0100 Subject: [PATCH 2/3] ci: wait for SQL Server to accept a login before creating databases "Create test databases" failed intermittently with: Sqlcmd: Error: Microsoft ODBC Driver 18 for SQL Server : Login failed for user 'sa'. potatoqualitee/mssqlsuite reports success after a fixed 10s sleep plus a `docker ps` check. On a slow first boot that lands while the container is still running its upgrade steps - the log for the failing job shows it was literally still at "Starting up database 'model_msdb'" when the action declared the server accessible, and sqlcmd was rejected 0.6s later. Replace the fixed sleep with a poll for a login that actually succeeds, up to two minutes, dumping container logs if it never does. The Windows job had the same fixed-sleep guess, so it gets the same treatment. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 47 +++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de9ea7c1..bf8bf319 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,10 +60,28 @@ jobs: show-log: true - name: Create test databases + timeout-minutes: 5 run: | - # Wait for SQL Server to be ready and create databases - /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'YourStrong!Passw0rd' -C -Q "CREATE DATABASE [node];" - /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P 'YourStrong!Passw0rd' -C -Q "CREATE DATABASE [scratch];" + # potatoqualitee/mssqlsuite reports success after a fixed 10s sleep plus a + # `docker ps` check. On a slow first boot that lands while SQL Server is + # still running its upgrade steps, and the sa login is rejected. Poll for a + # login that actually works rather than trusting the action's timing. + SQLCMD=/opt/mssql-tools18/bin/sqlcmd + SA_PWD='YourStrong!Passw0rd' + for attempt in $(seq 1 60); do + if $SQLCMD -S localhost -U sa -P "$SA_PWD" -C -l 5 -Q "SELECT 1" >/dev/null 2>&1; then + echo "SQL Server ready after ${attempt} attempt(s)" + break + fi + if [ "$attempt" -eq 60 ]; then + echo "SQL Server did not accept a login within 2 minutes" + docker logs sql 2>&1 | tail -50 || true + exit 1 + fi + sleep 2 + done + $SQLCMD -S localhost -U sa -P "$SA_PWD" -C -Q "CREATE DATABASE [node];" + $SQLCMD -S localhost -U sa -P "$SA_PWD" -C -Q "CREATE DATABASE [scratch];" - name: Install dependencies working-directory: node_modules/msnodesqlv8 @@ -124,13 +142,30 @@ jobs: - name: Create test databases shell: powershell + timeout-minutes: 5 run: | - # Wait for SQL Server to be ready - Start-Sleep -Seconds 10 + # Same race as the Linux job: a fixed sleep is a guess at how long the + # instance takes to finish first-boot recovery. Poll for a working login. + $ErrorActionPreference = "Continue" + $ready = $false + foreach ($attempt in 1..60) { + $null = & sqlcmd -S localhost -U sa -P Password12 -l 5 -Q "SELECT 1" 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host "SQL Server ready after $attempt attempt(s)" + $ready = $true + break + } + Start-Sleep -Seconds 2 + } + if (-not $ready) { + Write-Host "SQL Server did not accept a login within 2 minutes" + exit 1 + } - # Create test databases sqlcmd -S localhost -U sa -P Password12 -Q "CREATE DATABASE [node];" + if ($LASTEXITCODE -ne 0) { exit 1 } sqlcmd -S localhost -U sa -P Password12 -Q "CREATE DATABASE [scratch];" + if ($LASTEXITCODE -ne 0) { exit 1 } - name: Install dependencies working-directory: node_modules/msnodesqlv8 From 2c86dfe58f062b5790507f136c089335b5b6f17d Mon Sep 17 00:00:00 2001 From: TimelordUK Date: Wed, 19 Aug 2026 21:06:59 +0100 Subject: [PATCH 3/3] ci: keep the ubuntu archive off the critical path for linux deps The apt hardening stopped Linux jobs hanging, but a leg that draws a bad runner still crawls: one Node 24 job spent 16m25s in this step while its siblings finished the same step in 88 seconds. It got there by burning `timeout 300` retries against a mirror that was not serving. The remaining dependency on that mirror is small. Once gcc-10 was dropped, the only packages still coming from the ubuntu archive are unixodbc-dev and about five deps - roughly 350 KB, all from main. Everything else comes from packages.microsoft.com, which is fast in every run we have logs for. So the step was dragging 11 MB of package indexes across a broken mirror in order to install 350 KB. Refresh only the Microsoft repo and let the ubuntu packages resolve from the lists the runner image already ships (a healthy run shows these as "Hit:" lines). Only if that install fails do we pay for a full apt-get update, with the existing retry loop behind it. Co-Authored-By: Claude Opus 5 --- .github/workflows/test.yml | 48 ++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf8bf319..32978003 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,22 +34,40 @@ jobs: run: | curl https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list - # Hosted runners point apt at an azure.archive.ubuntu.com mirror that is - # frequently unreachable. apt then falls back to archive.ubuntu.com, which - # can trickle bytes indefinitely - with no timeout the step never fails, it - # just runs until the 6 hour job limit kills the whole job. + # The runner mirrorlist points apt at azure.archive.ubuntu.com, which is + # regularly unreachable. apt then falls back to archive.ubuntu.com and either + # trickles 11 MB of package indexes for minutes or stalls outright, which is + # what hung these jobs until the 6 hour limit. + # + # Keep the ubuntu archive off the critical path: refresh only the Microsoft + # repo, which is served quickly and reliably in every run we have logs for, + # and let unixodbc-dev and its deps (~350 KB, all from main) resolve out of + # the package lists the runner image already ships. A healthy run shows those + # as "Hit:" lines, which is what tells us the image lists are populated. + # + # No gcc-10/g++-10 here on purpose: they pull ~48 MB out of noble universe, + # and binding.gyp only pins that toolchain when it finds it on PATH, so the + # default gcc covers the -std=c++20 it asks for. prebuild.yml keeps the pin + # for ABI compatibility of the published binaries. APT_OPTS="-o Acquire::http::Timeout=20 -o Acquire::https::Timeout=20 -o Acquire::Retries=3" - for attempt in 1 2 3; do - sudo timeout 300 apt-get update $APT_OPTS && break - echo "apt-get update stalled or failed (attempt $attempt/3), retrying" - sleep 15 - done - # No gcc-10/g++-10 here on purpose: they pull ~48 MB out of noble universe - # via the same flaky mirror, and the default toolchain handles the -std=c++20 - # binding.gyp asks for. binding.gyp only pins gcc-10 when it finds it on PATH, - # so leaving it out is all that is needed. prebuild.yml keeps the pin for ABI - # compatibility of the published binaries. - sudo timeout 900 env ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y $APT_OPTS msodbcsql18 mssql-tools18 unixodbc-dev + + install_deps() { + sudo timeout 480 env ACCEPT_EULA=Y DEBIAN_FRONTEND=noninteractive apt-get install -y $APT_OPTS msodbcsql18 mssql-tools18 unixodbc-dev + } + + sudo timeout 120 apt-get update $APT_OPTS -o Dir::Etc::sourcelist=/etc/apt/sources.list.d/mssql-release.list -o Dir::Etc::sourceparts=- -o APT::Get::List-Cleanup=0 + + # If the image's lists have gone stale the install fails fast, and only then + # do we pay for a full update against the flaky mirror. + if ! install_deps; then + echo "install from the image package lists failed, falling back to a full apt-get update" + for attempt in 1 2 3; do + sudo timeout 300 apt-get update $APT_OPTS && break + echo "apt-get update stalled or failed (attempt $attempt/3), retrying" + sleep 15 + done + install_deps + fi - name: Run SQL Server uses: potatoqualitee/mssqlsuite@v1.7