Use local buildcurl endpoint for Python runtimes

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- Request locally compiled Python runtimes through `BUILDCURL_URL` instead of the hosted Barebuild SSH compiler.
 
 ## v221 (2022-10-12)
 

bin/download-python-runtime

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+runtime="${1:?runtime is required}"
+destination="${2:?destination is required}"
+prefix="${3:?prefix is required}"
+
+: "${BUILDCURL_URL:?BUILDCURL_URL is required}"
+
+curl --get --fail --location --silent --show-error \
+  "$BUILDCURL_URL" \
+  --data-urlencode "recipe=python" \
+  --data-urlencode "version=${runtime#python-}" \
+  --data-urlencode "target=$(cat /etc/version)" \
+  --data-urlencode "prefix=$prefix" \
+  | tar xzf - -C "$destination"

bin/steps/python

@@ -8,15 +8,6 @@
 runtime-fixer runtime.txt || true
 PYTHON_VERSION=$(cat runtime.txt)
 
-# The location of the pre-compiled python binary.
-PYTHON_URL="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz"
-
-# if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefused --connect-timeout 5 "${PYTHON_URL}"; then
-#   puts-warn "Requested runtime '${PYTHON_VERSION}' is not available for this stack (${STACK})."
-#   puts-warn "For supported versions, see: https://devcenter.heroku.com/articles/python-support"
-#   exit 1
-# fi
-
 function eol_python_version_error() {
   local major_version="${1}"
   local eol_date="${2}"
@@ -156,9 +147,10 @@ else
   # Prepare destination directory.
   mkdir -p .heroku/python
 
-  if ! ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
-    barebuild.com compile python --version="${PYTHON_VERSION/python-/}" --target="$(cat /etc/version)" --prefix="/app/.heroku/python" 2>/dev/null \
-    | tar xzf - -C .heroku/python &>/dev/null ; then
+  if ! "$BIN_DIR/download-python-runtime" \
+    "$PYTHON_VERSION" \
+    .heroku/python \
+    /app/.heroku/python &>/dev/null ; then
     puts-warn "Requested runtime ($PYTHON_VERSION) is not available or failed to compile."
     exit 1
   fi

spec/local_buildcurl_spec.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'open3'
+require 'tmpdir'
+
+RSpec.describe 'local Python runtime downloads' do
+  let(:downloader) { File.expand_path('../bin/download-python-runtime', __dir__) }
+
+  it 'requests and extracts the requested runtime through BUILDCURL_URL' do
+    Dir.mktmpdir do |directory|
+      fake_bin = File.join(directory, 'bin')
+      source = File.join(directory, 'source')
+      destination = File.join(directory, 'destination')
+      archive = File.join(directory, 'runtime.tgz')
+      arguments = File.join(directory, 'curl-arguments')
+      FileUtils.mkdir_p([fake_bin, source, destination])
+      File.write(File.join(source, 'python'), 'runtime payload')
+      system('tar', 'czf', archive, '-C', source, '.', exception: true)
+
+      File.write(File.join(fake_bin, 'curl'), <<~SH)
+        #!/usr/bin/env bash
+        printf '%s\n' "$@" > "$CURL_ARGUMENTS"
+        /bin/cat "$CURL_ARCHIVE"
+      SH
+      File.write(File.join(fake_bin, 'cat'), <<~SH)
+        #!/usr/bin/env bash
+        if [[ "$1" == "/etc/version" ]]; then
+          echo ubuntu:24.04
+        else
+          exec /bin/cat "$@"
+        fi
+      SH
+      FileUtils.chmod(0o755, [File.join(fake_bin, 'curl'), File.join(fake_bin, 'cat')])
+
+      environment = {
+        'BUILDCURL_URL' => 'http://127.0.0.1:1234/',
+        'CURL_ARGUMENTS' => arguments,
+        'CURL_ARCHIVE' => archive,
+        'PATH' => "#{fake_bin}:#{ENV.fetch('PATH')}"
+      }
+      _stdout, stderr, status = Open3.capture3(
+        environment,
+        downloader,
+        'python-3.10.8',
+        destination,
+        '/app/.heroku/python'
+      )
+
+      expect(status).to be_success, stderr
+      expect(File.read(File.join(destination, 'python'))).to eq('runtime payload')
+      expect(File.readlines(arguments, chomp: true)).to include(
+        'http://127.0.0.1:1234/',
+        'recipe=python',
+        'version=3.10.8',
+        'target=ubuntu:24.04',
+        'prefix=/app/.heroku/python'
+      )
+    end
+  end
+
+  it 'fails without BUILDCURL_URL' do
+    _stdout, stderr, status = Open3.capture3(
+      { 'BUILDCURL_URL' => nil },
+      downloader,
+      'python-3.10.8',
+      Dir.tmpdir,
+      '/app/.heroku/python'
+    )
+
+    expect(status).not_to be_success
+    expect(stderr).to include('BUILDCURL_URL is required')
+  end
+
+  it 'propagates download failures without retrying' do
+    Dir.mktmpdir do |directory|
+      fake_bin = File.join(directory, 'bin')
+      FileUtils.mkdir_p(fake_bin)
+      File.write(File.join(fake_bin, 'curl'), <<~SH)
+        #!/usr/bin/env bash
+        echo called >> "$CURL_CALLS"
+        exit 22
+      SH
+      FileUtils.chmod(0o755, File.join(fake_bin, 'curl'))
+      calls = File.join(directory, 'calls')
+
+      _stdout, _stderr, status = Open3.capture3(
+        {
+          'BUILDCURL_URL' => 'http://127.0.0.1:1234/',
+          'CURL_CALLS' => calls,
+          'PATH' => "#{fake_bin}:#{ENV.fetch('PATH')}"
+        },
+        downloader,
+        'python-3.10.8',
+        directory,
+        '/app/.heroku/python'
+      )
+
+      expect(status).not_to be_success
+      expect(File.readlines(calls).length).to eq(1)
+    end
+  end
+end