From dbcaa2fb6378f62860607246ede742fb30f96d80 Mon Sep 17 00:00:00 2001 From: pdparchitect Date: Fri, 4 Sep 2026 20:55:41 +0000 Subject: [PATCH] refactor: update storage configuration variables and documentation for clarity (+1 more) - refactor: update storage configuration variables and documentation for clarity - feat: enhance GHCR publish workflow to support image reuse and optimize build process --- .github/workflows/publish-ghcr-platform.yaml | 121 ++++++++++++++--- docker-compose.yml | 36 +++-- docker/distro/community/compose.yml | 41 +++--- docker/entrypoint.sh | 126 +++++++++++++++--- docker/garage/garage.toml | 2 +- docker/garage/init.mjs | 101 +++++++++++++- docs/deployment.md | 39 +++--- docs/getting-started.md | 25 ++-- packages/storage/README.md | 15 ++- packages/storage/src/adapter.test.js | 83 +++++++++++- packages/storage/src/index.ts | 69 +++++++--- packages/storage/src/mounts.test.js | 16 +-- packages/storage/src/polyfill.test.js | 6 +- packages/storage/src/sts.ts | 18 +-- platform/.env.example | 18 +-- platform/config/models.ts | 44 ++++++ platform/lib/action.exec.file.utest.js | 6 +- platform/lib/model.provider.openai.adaptor.ts | 6 + .../model.provider.openai.adaptor.utest.js | 10 ++ .../next.config.d/bundling.agentos.config.js | 37 +++++ .../bundling.agentos.config.utest.js | 24 ++++ platform/next.config.d/bundling.config.js | 20 --- 22 files changed, 693 insertions(+), 170 deletions(-) create mode 100644 platform/next.config.d/bundling.agentos.config.js create mode 100644 platform/next.config.d/bundling.agentos.config.utest.js diff --git a/.github/workflows/publish-ghcr-platform.yaml b/.github/workflows/publish-ghcr-platform.yaml index 4c70153..a534119 100644 --- a/.github/workflows/publish-ghcr-platform.yaml +++ b/.github/workflows/publish-ghcr-platform.yaml @@ -35,6 +35,13 @@ jobs: # here from the actual diff of the push. Anything that prevents computing # that diff (a brand-new branch, a force push, a manual dispatch) falls back # to building - a spurious build is cheap, a silently skipped one is not. + # + # A build-relevant push whose tree was already built is not rebuilt either: + # main only moves through the promotion pull request (next -> main), so its + # merge commit carries the exact tree of a next head that build, verify and + # publish already tagged as sha-. That tag is reused and retagged by + # the publish job instead of spending two 8-core runners on an identical + # image. Any tree that no published tag matches falls back to a full build. changes: if: >- (github.repository == 'chatbotkit/platform' || startsWith(github.repository, 'chatbotkit/platform-')) && @@ -43,6 +50,7 @@ jobs: runs-on: ubuntu-latest outputs: build: ${{ steps.diff.outputs.build }} + reuse: ${{ steps.reuse.outputs.reuse }} steps: - uses: actions/checkout@v7 @@ -70,6 +78,72 @@ jobs: echo "build=false" >> "$GITHUB_OUTPUT" fi + # @note reuse is decided once for the single flavor; a second flavor + # needs a per-flavor lookup and output here + - name: Resolve image names + if: steps.diff.outputs.build == 'true' + id: image + env: + FLAVOR: community + REPOSITORY_NAME: ${{ github.event.repository.name }} + REPOSITORY_OWNER: ${{ github.repository_owner }} + run: | + owner="${REPOSITORY_OWNER,,}" + repository="${REPOSITORY_NAME,,}" + stack="${REGISTRY}/${owner}/${repository}-${FLAVOR}" + echo "application=${stack}-app" >> "$GITHUB_OUTPUT" + echo "initializer=${stack}-init" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + if: steps.diff.outputs.build == 'true' + uses: docker/setup-buildx-action@v4 + + - name: Log in to GitHub Container Registry + if: steps.diff.outputs.build == 'true' + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # @note candidates are the pushed commit (a fast-forward or a re-run) + # and its parents (the promotion merge, whose second parent is the next + # head). A candidate counts only when its tree is byte-identical to the + # pushed tree and both component images carry its sha- tag, which only + # publish creates and only after verify passed + - name: Find a published image of the same tree + if: steps.diff.outputs.build == 'true' + id: reuse + env: + APPLICATION_IMAGE: ${{ steps.image.outputs.application }} + INITIALIZER_IMAGE: ${{ steps.image.outputs.initializer }} + run: | + tree=$(git rev-parse "${GITHUB_SHA}^{tree}") + + # @note the checkout is shallow, so parents are read from the raw + # commit object; rev-list would report the grafted commit as a root + parents=$(git cat-file -p "$GITHUB_SHA" | awk '/^parent /{print $2}') + + for candidate in "$GITHUB_SHA" $parents; do + if [ "$candidate" != "$GITHUB_SHA" ]; then + git fetch --quiet --depth=1 origin "$candidate" || continue + fi + + [ "$(git rev-parse "${candidate}^{tree}")" = "$tree" ] || continue + + short="${candidate:0:7}" + + if docker buildx imagetools inspect "${APPLICATION_IMAGE}:sha-${short}" >/dev/null 2>&1 \ + && docker buildx imagetools inspect "${INITIALIZER_IMAGE}:sha-${short}" >/dev/null 2>&1 + then + echo "Reusing images built from ${candidate}" + echo "reuse=${short}" >> "$GITHUB_OUTPUT" + exit 0 + fi + done + + echo "reuse=" >> "$GITHUB_OUTPUT" + # @note `next` receives direct pushes, so nothing has vetted the code yet - # the quality gate runs alongside the image build and blocks publication, # not the build itself: build only pushes untagged per-architecture digests, @@ -95,6 +169,7 @@ jobs: !cancelled() && (github.repository == 'chatbotkit/platform' || startsWith(github.repository, 'chatbotkit/platform-')) && needs.changes.outputs.build == 'true' && + needs.changes.outputs.reuse == '' && github.actor != 'github-actions[bot]' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/next') name: Build ${{ matrix.flavor.name }} (${{ matrix.architecture.name }}) @@ -283,16 +358,18 @@ jobs: # leaves the build's untagged digests unreachable in the registry and # publishes nothing. !cancelled() + accepting the skipped verify is # required: the implicit success() looks at the whole needs chain, and - # verify is skipped on main + # verify is skipped on main. A skipped build is accepted only when the + # changes job found a published sha- tag of the same tree to retag. if: >- !cancelled() && (github.repository == 'chatbotkit/platform' || startsWith(github.repository, 'chatbotkit/platform-')) && - needs.build.result == 'success' && + (needs.build.result == 'success' || (needs.build.result == 'skipped' && needs.changes.outputs.reuse != '')) && (needs.verify.result == 'success' || needs.verify.result == 'skipped') && github.actor != 'github-actions[bot]' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/next') name: Publish ${{ matrix.flavor.name }} needs: + - changes - build - verify runs-on: ${{ matrix.flavor.runner }} @@ -339,12 +416,16 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Download image digests + if: needs.build.result == 'success' uses: actions/download-artifact@v7 with: pattern: platform-digests-${{ matrix.flavor.name }}-* path: ${{ runner.temp }}/platform-digests merge-multiple: true + # @note with a reused build the source is the already multi-platform + # sha- manifest list, which imagetools copies under the new tags; a + # fresh build supplies one per-architecture digest per image instead - name: Create multi-platform image manifests id: manifest env: @@ -352,22 +433,28 @@ jobs: CHANNEL: ${{ github.ref_name }} DIGESTS_DIR: ${{ runner.temp }}/platform-digests INITIALIZER_IMAGE: ${{ steps.image.outputs.initializer }} + REUSE: ${{ needs.changes.outputs.reuse }} run: | - application_sources=() - for digest_file in "$DIGESTS_DIR"/application/*; do - [ -f "$digest_file" ] || continue - application_sources+=("${APPLICATION_IMAGE}@sha256:$(basename "$digest_file")") - done - - initializer_sources=() - for digest_file in "$DIGESTS_DIR"/initializer/*; do - [ -f "$digest_file" ] || continue - initializer_sources+=("${INITIALIZER_IMAGE}@sha256:$(basename "$digest_file")") - done - - if [ "${#application_sources[@]}" -ne 2 ] || [ "${#initializer_sources[@]}" -ne 2 ]; then - echo "Expected two architecture digests for each image" >&2 - exit 1 + if [ -n "$REUSE" ]; then + application_sources=("${APPLICATION_IMAGE}:sha-${REUSE}") + initializer_sources=("${INITIALIZER_IMAGE}:sha-${REUSE}") + else + application_sources=() + for digest_file in "$DIGESTS_DIR"/application/*; do + [ -f "$digest_file" ] || continue + application_sources+=("${APPLICATION_IMAGE}@sha256:$(basename "$digest_file")") + done + + initializer_sources=() + for digest_file in "$DIGESTS_DIR"/initializer/*; do + [ -f "$digest_file" ] || continue + initializer_sources+=("${INITIALIZER_IMAGE}@sha256:$(basename "$digest_file")") + done + + if [ "${#application_sources[@]}" -ne 2 ] || [ "${#initializer_sources[@]}" -ne 2 ]; then + echo "Expected two architecture digests for each image" >&2 + exit 1 + fi fi short_sha="${GITHUB_SHA:0:7}" diff --git a/docker-compose.yml b/docker-compose.yml index cd5934d..a675c4b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,17 +37,21 @@ # is Garage, spoken to over the plain S3 protocol - see docker/garage/. # The credentials are the known development values garage-init provisions. # -# @note presigned upload/download URLs carry this endpoint, so a browser on -# the host needs to resolve it too: add `127.0.0.1 garage` to /etc/hosts to -# use browser-facing file flows with the containerized modes. (When running -# `pnpm dev` on the host instead, point SERVICE_AWS_ENDPOINT at -# http://localhost:3900 in platform/.env and the question does not arise.) +# @note the store is a separate, published service: the application reaches +# it as garage:3900 inside the network, while browsers upload and download +# through presigned URLs minted against STORAGE_PUBLIC_ENDPOINT - the +# store's own name from the browser's point of view. Like the relay and the +# app shells it gets a `*.localhost` name, which browsers resolve to loopback +# with no DNS setup; set STORAGE_URL when the browser reaches the machine by +# another address. (When running `pnpm dev` on the host instead, +# STORAGE_ENDPOINT=http://localhost:3900 in platform/.env serves both roles.) x-storage-env: &storage-env - SERVICE_AWS_ENDPOINT: http://garage:3900 - SERVICE_AWS_REGION: garage - SERVICE_AWS_ACCESS_KEY_ID: GK31e57eba9df26b2e7e1b0eaa - SERVICE_AWS_SECRET_ACCESS_KEY: 9f3c1e2b8a4d5f6071829304a5b6c7d8e9f00112233445566778899aabbccdde - SERVICE_AWS_FORCE_PATH_STYLE: 'true' + STORAGE_ENDPOINT: http://garage:3900 + STORAGE_PUBLIC_ENDPOINT: ${STORAGE_URL:-http://cbk-storage.localhost:${STORAGE_PORT:-3900}} + STORAGE_REGION: garage + STORAGE_ACCESS_KEY_ID: GK31e57eba9df26b2e7e1b0eaa + STORAGE_SECRET_ACCESS_KEY: 9f3c1e2b8a4d5f6071829304a5b6c7d8e9f00112233445566778899aabbccdde + STORAGE_FORCE_PATH_STYLE: 'true' FILE_S3_BUCKET_NAME: file IMAGE_S3_BUCKET_NAME: image VIDEO_S3_BUCKET_NAME: video @@ -250,10 +254,10 @@ services: # warnings and errors RUST_LOG: warn ports: - # @note published on localhost only, so `pnpm dev` on the host can use - # this same store (SERVICE_AWS_ENDPOINT=http://localhost:3900) without - # building anything - - '127.0.0.1:3900:3900' + # @note published on every interface: browsers talk to the store + # directly through presigned URLs (see x-storage-env), and `pnpm dev` + # on the host uses the same port + - '${STORAGE_PORT:-3900}:3900' volumes: - ./docker/garage/garage.toml:/etc/garage.toml:ro - garage-data:/var/lib/garage @@ -275,6 +279,10 @@ services: environment: GARAGE_ADMIN_URL: http://garage:3903 GARAGE_ADMIN_TOKEN: dev-admin-token + GARAGE_S3_URL: http://garage:3900 + # @note origins allowed to use presigned URLs from a browser; the URLs + # themselves are the access control + STORAGE_CORS_ORIGINS: ${STORAGE_CORS_ORIGINS:-*} STORAGE_ACCESS_KEY_ID: GK31e57eba9df26b2e7e1b0eaa STORAGE_SECRET_ACCESS_KEY: 9f3c1e2b8a4d5f6071829304a5b6c7d8e9f00112233445566778899aabbccdde volumes: diff --git a/docker/distro/community/compose.yml b/docker/distro/community/compose.yml index 883350f..158bef2 100644 --- a/docker/distro/community/compose.yml +++ b/docker/distro/community/compose.yml @@ -30,15 +30,20 @@ # volume (garage-init provisions it, the application entrypoint sources it); # set STORAGE_ACCESS_KEY_ID / STORAGE_SECRET_ACCESS_KEY to use a fixed pair. # -# @note presigned upload/download URLs carry this endpoint, so a browser on -# the host needs to resolve it too: add `127.0.0.1 garage` to /etc/hosts to -# use browser-facing file flows. +# @note the store is a separate, published service: the application reaches +# it as garage:3900 inside the network, while browsers upload and download +# through presigned URLs minted against STORAGE_PUBLIC_ENDPOINT - the +# store's own name from the browser's point of view. Like the relay and the +# app shells it gets a `*.localhost` name, which browsers resolve to loopback +# with no DNS setup; set STORAGE_URL to an address the browser can reach (and +# TLS if the site has it) when that is not the machine itself. x-storage-env: &storage-env - SERVICE_AWS_ENDPOINT: http://garage:3900 - SERVICE_AWS_REGION: garage - SERVICE_AWS_ACCESS_KEY_ID: ${STORAGE_ACCESS_KEY_ID:-} - SERVICE_AWS_SECRET_ACCESS_KEY: ${STORAGE_SECRET_ACCESS_KEY:-} - SERVICE_AWS_FORCE_PATH_STYLE: 'true' + STORAGE_ENDPOINT: http://garage:3900 + STORAGE_PUBLIC_ENDPOINT: ${STORAGE_URL:-http://cbk-storage.localhost:${STORAGE_PORT:-3900}} + STORAGE_REGION: garage + STORAGE_ACCESS_KEY_ID: ${STORAGE_ACCESS_KEY_ID:-} + STORAGE_SECRET_ACCESS_KEY: ${STORAGE_SECRET_ACCESS_KEY:-} + STORAGE_FORCE_PATH_STYLE: 'true' FILE_S3_BUCKET_NAME: file IMAGE_S3_BUCKET_NAME: image VIDEO_S3_BUCKET_NAME: video @@ -100,7 +105,8 @@ services: # instead of a .env file, prompted for or set directly: # docker compose -f oci://... run --rm --no-deps platform setup # docker compose -f oci://... run --rm --no-deps platform setup OPENROUTER_MODELS_API_KEY=... - # Values given here or in .env win over persisted ones - see + # Values given here or in .env win over persisted ones, and a running + # service restarts itself when the persisted file changes - see # docker/entrypoint.sh. An override file remains the other route: # docker compose -f oci://... -f my-override.yml up -d OPENAI_API_KEY: ${OPENAI_API_KEY:-} @@ -174,10 +180,9 @@ services: # warnings and errors RUST_LOG: warn ports: - # @note published on localhost so presigned URLs (which carry the - # garage:3900 endpoint) work from a host browser with the /etc/hosts - # entry described above - - '127.0.0.1:3900:3900' + # @note published on every interface: browsers talk to the store + # directly through presigned URLs (see x-storage-env) + - '${STORAGE_PORT:-3900}:3900' configs: - source: garage-config target: /etc/garage.toml @@ -203,6 +208,10 @@ services: environment: GARAGE_ADMIN_URL: http://garage:3903 GARAGE_ADMIN_TOKEN: ${GARAGE_ADMIN_TOKEN:-dev-admin-token} + GARAGE_S3_URL: http://garage:3900 + # @note origins allowed to use presigned URLs from a browser; the URLs + # themselves are the access control + STORAGE_CORS_ORIGINS: ${STORAGE_CORS_ORIGINS:-*} STORAGE_ACCESS_KEY_ID: ${STORAGE_ACCESS_KEY_ID:-} STORAGE_SECRET_ACCESS_KEY: ${STORAGE_SECRET_ACCESS_KEY:-} volumes: @@ -222,8 +231,8 @@ configs: # Garage (S3-compatible object storage) - single-node configuration. # # WARNING: the rpc_secret and admin token default to known development - # values. Neither port is published outside the compose network, but a - # hardened setup overrides them (GARAGE_RPC_SECRET, `openssl rand -hex + # values. Neither the RPC nor the admin port is published outside the + # compose network (the S3 port is), but a hardened setup overrides them (GARAGE_RPC_SECRET, `openssl rand -hex # 32`, and GARAGE_ADMIN_TOKEN) - and a real deployment almost certainly # runs a real store with replication rather than this single-node # layout. @@ -239,7 +248,7 @@ configs: rpc_secret = "${GARAGE_RPC_SECRET:-1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec}" [s3_api] - # @note the region is part of every SigV4 signature: SERVICE_AWS_REGION + # @note the region is part of every SigV4 signature: STORAGE_REGION # must match it, or every request fails authentication s3_region = "garage" api_bind_addr = "[::]:3900" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 9937b89..7a4f281 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -9,14 +9,25 @@ set -e # 3. $DATA_DIR/.secrets.env - values generated on first boot # An empty environment value counts as unset, so a Compose default such as # `${OPENAI_API_KEY:-}` never masks a persisted value. +# +# The command runs as a child of this script, which polls config.env every +# PLATFORM_CONFIG_WATCH_INTERVAL seconds (default 5) and restarts the child +# when the file changes, so `setup` applies without an operator restart. Set +# the interval to 0 to exec the command directly instead. DATA_DIR="${PLATFORM_DATA_DIR:-/data}" CONFIG_FILE="$DATA_DIR/config.env" SECRETS_FILE="$DATA_DIR/.secrets.env" STORAGE_FILE="$DATA_DIR/.storage.env" +WATCH_INTERVAL="${PLATFORM_CONFIG_WATCH_INTERVAL:-5}" DEFAULT_SETUP_KEYS="OPENAI_API_KEY OPENROUTER_MODELS_API_KEY VERCEL_MODELS_API_KEY" +# @note every variable this script exports is recorded so a reload can unset +# them all and resolve from scratch; container environment values are never +# in the list, which is what keeps them winning +LOADED_KEYS="" + is_empty() { eval "[ -z \"\${$1}\" ]" } @@ -27,6 +38,11 @@ is_valid_key() { esac } +load_export() { + export "$1=$2" + LOADED_KEYS="$LOADED_KEYS $1" +} + config_get() { [ -f "$CONFIG_FILE" ] || return 0 sed -n "s/^$1=//p" "$CONFIG_FILE" | tail -n 1 @@ -55,11 +71,16 @@ config_load() { value="${line#*=}" is_valid_key "$key" || continue if is_empty "$key"; then - export "$key=$value" + load_export "$key" "$value" fi done < "$CONFIG_FILE" } +config_fingerprint() { + [ -f "$CONFIG_FILE" ] || return 0 + cksum < "$CONFIG_FILE" +} + # read_secret KEY - prompts on the terminal without echo; Enter keeps the # current value, a single "-" clears it read_secret() { @@ -109,7 +130,7 @@ run_setup() { ;; esac done - echo "INFO: values persisted in $CONFIG_FILE - restart the platform service to apply" >&2 + echo "INFO: values persisted in $CONFIG_FILE - a running platform service restarts itself to apply them; restart it by hand if config watching is disabled" >&2 } if [ "$1" = "setup" ]; then @@ -118,13 +139,15 @@ if [ "$1" = "setup" ]; then exit 0 fi -config_load - # Fills empty NEXTAUTH_SECRET / QUEUE_SECRET / JWT_TOKEN_SECRET_KEY / # CLOAK_ENCRYPTION_KEY with values generated once and persisted in $DATA_DIR, # so sessions, queue signatures, issued tokens and encrypted values survive # restarts as long as it is a volume. -if [ -z "$NEXTAUTH_SECRET" ] || [ -z "$QUEUE_SECRET" ] || [ -z "$JWT_TOKEN_SECRET_KEY" ] || [ -z "$CLOAK_ENCRYPTION_KEY" ]; then +secrets_load() { + if [ -n "$NEXTAUTH_SECRET" ] && [ -n "$QUEUE_SECRET" ] && [ -n "$JWT_TOKEN_SECRET_KEY" ] && [ -n "$CLOAK_ENCRYPTION_KEY" ]; then + return 0 + fi + generate_secret() { node -e 'process.stdout.write(require("node:crypto").randomBytes(32).toString("hex"))' } @@ -158,34 +181,105 @@ if [ -z "$NEXTAUTH_SECRET" ] || [ -z "$QUEUE_SECRET" ] || [ -z "$JWT_TOKEN_SECRE if [ -z "$NEXTAUTH_SECRET" ]; then echo "WARNING: NEXTAUTH_SECRET is not set - using a generated value persisted in $SECRETS_FILE" >&2 - export NEXTAUTH_SECRET="$GENERATED_NEXTAUTH_SECRET" + load_export NEXTAUTH_SECRET "$GENERATED_NEXTAUTH_SECRET" fi if [ -z "$QUEUE_SECRET" ]; then echo "WARNING: QUEUE_SECRET is not set - using a generated value persisted in $SECRETS_FILE" >&2 - export QUEUE_SECRET="$GENERATED_QUEUE_SECRET" + load_export QUEUE_SECRET "$GENERATED_QUEUE_SECRET" fi if [ -z "$JWT_TOKEN_SECRET_KEY" ]; then echo "WARNING: JWT_TOKEN_SECRET_KEY is not set - using a generated value persisted in $SECRETS_FILE" >&2 - export JWT_TOKEN_SECRET_KEY="$GENERATED_JWT_TOKEN_SECRET_KEY" + load_export JWT_TOKEN_SECRET_KEY "$GENERATED_JWT_TOKEN_SECRET_KEY" fi if [ -z "$CLOAK_ENCRYPTION_KEY" ]; then echo "WARNING: CLOAK_ENCRYPTION_KEY is not set - using a generated value persisted in $SECRETS_FILE" >&2 - export CLOAK_ENCRYPTION_KEY="$GENERATED_CLOAK_ENCRYPTION_KEY" + load_export CLOAK_ENCRYPTION_KEY "$GENERATED_CLOAK_ENCRYPTION_KEY" fi -fi +} # Storage credentials generated by garage-init land in the shared data # volume - see docker/garage/init.mjs. -if [ -z "$SERVICE_AWS_ACCESS_KEY_ID" ] && [ -f "$STORAGE_FILE" ]; then - . "$STORAGE_FILE" +storage_load() { + if [ -z "$STORAGE_ACCESS_KEY_ID" ] && [ -f "$STORAGE_FILE" ]; then + . "$STORAGE_FILE" + + echo "INFO: using generated storage credentials from $STORAGE_FILE" >&2 + + load_export STORAGE_ACCESS_KEY_ID "$GENERATED_STORAGE_ACCESS_KEY_ID" + load_export STORAGE_SECRET_ACCESS_KEY "$GENERATED_STORAGE_SECRET_ACCESS_KEY" + fi +} + +runtime_load() { + config_load + secrets_load + storage_load +} - echo "INFO: using generated storage credentials from $STORAGE_FILE" >&2 +runtime_unload() { + for key in $LOADED_KEYS; do + unset "$key" + done + LOADED_KEYS="" +} + +runtime_load - export SERVICE_AWS_ACCESS_KEY_ID="$GENERATED_STORAGE_ACCESS_KEY_ID" - export SERVICE_AWS_SECRET_ACCESS_KEY="$GENERATED_STORAGE_SECRET_ACCESS_KEY" +if [ "$WATCH_INTERVAL" = "0" ]; then + exec "$@" fi -exec "$@" +# The command runs as a child; its own exit ends the container with its +# status, as `exec` would, so a crash still reaches the restart policy. Only a +# change to $CONFIG_FILE restarts it. +child="" +stopping="" + +forward_signal() { + stopping=1 + if [ -n "$child" ]; then + kill -s "$1" "$child" 2>/dev/null || true + fi +} + +trap 'forward_signal TERM' TERM +trap 'forward_signal INT' INT + +fingerprint="$(config_fingerprint)" + +while :; do + "$@" & + child=$! + + while :; do + # @note sleeping in the background keeps the wait interruptible, so a + # trapped signal is forwarded at once rather than after the tick + sleep "$WATCH_INTERVAL" & + wait $! || true + + if [ -n "$stopping" ]; then + break + fi + + kill -0 "$child" 2>/dev/null || break + + current="$(config_fingerprint)" + if [ "$current" != "$fingerprint" ]; then + fingerprint="$current" + echo "INFO: $CONFIG_FILE changed - restarting the command to apply it" >&2 + kill -s TERM "$child" 2>/dev/null || true + wait "$child" || true + child="" + runtime_unload + runtime_load + continue 2 + fi + done + + status=0 + wait "$child" || status=$? + exit "$status" +done diff --git a/docker/garage/garage.toml b/docker/garage/garage.toml index 143961b..7087574 100644 --- a/docker/garage/garage.toml +++ b/docker/garage/garage.toml @@ -23,7 +23,7 @@ rpc_public_addr = "127.0.0.1:3901" rpc_secret = "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" [s3_api] -# @note the region is part of every SigV4 signature: SERVICE_AWS_REGION must +# @note the region is part of every SigV4 signature: STORAGE_REGION must # match it, or every request fails authentication s3_region = "garage" api_bind_addr = "[::]:3900" diff --git a/docker/garage/init.mjs b/docker/garage/init.mjs index 39e7261..069088f 100644 --- a/docker/garage/init.mjs +++ b/docker/garage/init.mjs @@ -11,11 +11,26 @@ // // The bucket list mirrors the scopes in @chatbotkit-dev/storage; the compose // file carries the matching *_S3_BUCKET_NAME variables. +// +// Browsers upload and download through presigned URLs straight against the +// store, so every bucket also gets a CORS rule. Garage only takes CORS over +// the S3 API, hence the SigV4 signing below. +import { createHash, createHmac } from 'node:crypto' import { chownSync, readFileSync, writeFileSync } from 'node:fs' const base = process.env.GARAGE_ADMIN_URL const token = process.env.GARAGE_ADMIN_TOKEN +const s3Url = process.env.GARAGE_S3_URL || 'http://garage:3900' +const s3Region = process.env.GARAGE_S3_REGION || 'garage' + +// @note presigned URLs are the access control; the rule only lets the +// browser make the call. Narrow it for a store reachable beyond the host +const corsOrigins = (process.env.STORAGE_CORS_ORIGINS || '*') + .split(',') + .map((origin) => origin.trim()) + .filter(Boolean) + let accessKeyId = process.env.STORAGE_ACCESS_KEY_ID let secretAccessKey = process.env.STORAGE_SECRET_ACCESS_KEY @@ -141,6 +156,86 @@ if (!keys.some((key) => key.id === accessKeyId)) { console.log('[garage-init] access key imported') } +function sha256(data) { + return createHash('sha256').update(data).digest('hex') +} + +function hmac(key, data, encoding) { + return createHmac('sha256', key).update(data).digest(encoding) +} + +// @note minimal SigV4 for one path-style request - enough to avoid pulling +// the AWS SDK into the initializer image for a single call +async function s3(method, bucket, query, body) { + const url = new URL(`/${bucket}?${query}`, s3Url) + + const amzDate = new Date().toISOString().replace(/[-:]|\.\d{3}/g, '') + const date = amzDate.slice(0, 8) + const scope = `${date}/${s3Region}/s3/aws4_request` + const payloadHash = sha256(body) + + const headers = { + host: url.host, + 'x-amz-content-sha256': payloadHash, + 'x-amz-date': amzDate, + } + + const signedHeaders = Object.keys(headers).sort().join(';') + + const canonicalRequest = [ + method, + url.pathname, + `${query}=`, + ...Object.keys(headers) + .sort() + .map((name) => `${name}:${headers[name]}`), + '', + signedHeaders, + payloadHash, + ].join('\n') + + const stringToSign = [ + 'AWS4-HMAC-SHA256', + amzDate, + scope, + sha256(canonicalRequest), + ].join('\n') + + const signingKey = ['s3', 'aws4_request'].reduce( + (key, part) => hmac(key, part), + hmac(hmac(`AWS4${secretAccessKey}`, date), s3Region) + ) + + const signature = hmac(signingKey, stringToSign, 'hex') + + const response = await fetch(url, { + method, + body, + headers: { + ...headers, + authorization: `AWS4-HMAC-SHA256 Credential=${accessKeyId}/${scope}, SignedHeaders=${signedHeaders}, Signature=${signature}`, + 'content-type': 'application/xml', + }, + }) + + if (!response.ok) { + throw new Error( + `${method} /${bucket}?${query} -> ${response.status}: ${(await response.text()).slice(0, 200)}` + ) + } +} + +const corsConfiguration = + '' + + corsOrigins + .map((origin) => `${origin}`) + .join('') + + ['GET', 'PUT', 'HEAD'].map((m) => `${m}`).join('') + + '*' + + 'ETag' + + '3600' + + '' + const buckets = await api('/v2/ListBuckets') for (const alias of BUCKETS) { @@ -155,6 +250,10 @@ for (const alias of BUCKETS) { accessKeyId, permissions: { read: true, write: true, owner: true }, }) + + await s3('PUT', alias, 'cors', corsConfiguration) } -console.log(`[garage-init] buckets ready: ${BUCKETS.join(' ')}`) +console.log( + `[garage-init] buckets ready: ${BUCKETS.join(' ')} (cors: ${corsOrigins.join(' ')})` +) diff --git a/docs/deployment.md b/docs/deployment.md index 954bd9e..3eee6ca 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -110,14 +110,20 @@ docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest run --rm -- docker compose -f oci://ghcr.io/chatbotkit/platform-community:latest run --rm --no-deps platform setup OPENROUTER_MODELS_API_KEY=sk-or-... OPENAI_API_KEY= ``` -Restart the `platform` service afterwards. Precedence, highest first: the -container environment (shell, `.env`, `--env-file`, `-e`), then -`config.env`, then the secrets generated on first boot. An empty environment -value counts as unset, so the stack's `${OPENAI_API_KEY:-}` defaults never -mask a persisted value; to override one for a single run, set it in the -shell. The file is owned by the application user with mode `0600`; back it -up with the volume, and prefer `PRISMA_FIELD_ENCRYPTION_KEY` in the -environment rather than next to the database it protects. +A running `platform` service applies the change on its own: the entrypoint +polls `config.env` every `PLATFORM_CONFIG_WATCH_INTERVAL` seconds (default +5) and restarts the application process when it changes, so expect a short +interruption rather than a reload. Set the interval to `0` in an override +file to disable the watch and run the application as the container's main +process; the service then needs a `docker compose restart platform` after +each change. Precedence, highest first: the container environment (shell, +`.env`, `--env-file`, `-e`), then `config.env`, then the secrets generated +on first boot. An empty environment value counts as unset, so the stack's +`${OPENAI_API_KEY:-}` defaults never mask a persisted value; to override one +for a single run, set it in the shell. The file is owned by the application +user with mode `0600`; back it up with the volume, and prefer +`PRISMA_FIELD_ENCRYPTION_KEY` in the environment rather than next to the +database it protects. ### Several instances on one host @@ -136,7 +142,7 @@ services: - '3001:3000' garage: ports: !override - - '127.0.0.1:3901:3900' + - '3901:3900' ``` ```bash @@ -147,14 +153,12 @@ docker compose -p cbk-staging \ ``` Set `SITE_URL` and `NEXTAUTH_URL` to the instance's published address -(`http://localhost:3001` here) - in the shell or through `--env-file`, since a +(`http://localhost:3001` here) and `STORAGE_URL` to its store +(`http://cbk-storage.localhost:3901`) - in the shell or through `--env-file`, since a single `.env` in the working directory cannot describe both instances. Volumes, networks and container names are all prefixed with the project name, so each instance keeps its own database, generated secrets, object store and vector index, and `-p` is also how `logs`, `ps` and `down` find the right one. -Presigned storage URLs carry `garage:3900`, so the `/etc/hosts` entry serves the -instance that keeps port 3900; browser-facing file flows on the others need -their own store host name and endpoint. The artifact is published from [docker/distro/community/compose.yml](../docker/distro/community/compose.yml), @@ -164,8 +168,13 @@ under `docker/distro/`; a future PostgreSQL flavor publishes as matching image flavor. Browser-facing file upload and download flows presign URLs against the -in-stack store; add `127.0.0.1 garage` to `/etc/hosts` on the host to use -them, as with the development stack. +in-stack store, which the stack publishes on port 3900 (`STORAGE_PORT`) under +its own name: the URLs are minted against `STORAGE_PUBLIC_ENDPOINT`, +`http://cbk-storage.localhost:3900` by default, a `*.localhost` name browsers +resolve to loopback like the relay and app shells. Set `STORAGE_URL` to the +address browsers actually reach the host on (with TLS if the site has it). `garage-init` grants every bucket a CORS +rule for `STORAGE_CORS_ORIGINS` (default `*` - the presigned URL is the access +control; narrow it for a store reachable beyond the host). ### Distribution flavors diff --git a/docs/getting-started.md b/docs/getting-started.md index 1f1c789..19c3ddf 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -30,10 +30,11 @@ This command starts: The checkout is mounted read-only and synchronized into the development container. Editing the host working tree still triggers hot reload. -Browser-facing file flows use presigned URLs containing the Compose service -hostname. Add `127.0.0.1 garage` to the host machine's hosts file before testing -uploads or downloads in a containerized mode. Host-side development configured -with `SERVICE_AWS_ENDPOINT=http://localhost:3900` does not need that entry. +Browser-facing file flows use presigned URLs against the store itself, which +is published on port 3900 (`STORAGE_PORT`) under its own name, +`http://cbk-storage.localhost:3900` - a `*.localhost` name browsers resolve to +loopback with no DNS setup, like the relay and app shells. Set `STORAGE_URL` +when the browser reaches the machine by another address. No hosted account, billing configuration or vendor credential is required to boot. Model-backed agent responses require at least one model provider key. @@ -114,19 +115,19 @@ The quickest local option is the Compose Garage service: docker compose up garage garage-init ``` -It publishes a provisioned store on `127.0.0.1:3900`. Uncomment the matching -block in `.env.example`, including: +It publishes a provisioned store on port 3900. Uncomment the matching block in +`.env.example`, including: -- `SERVICE_AWS_ENDPOINT` -- `SERVICE_AWS_REGION` -- `SERVICE_AWS_ACCESS_KEY_ID` -- `SERVICE_AWS_SECRET_ACCESS_KEY` -- `SERVICE_AWS_FORCE_PATH_STYLE` +- `STORAGE_ENDPOINT` +- `STORAGE_REGION` +- `STORAGE_ACCESS_KEY_ID` +- `STORAGE_SECRET_ACCESS_KEY` +- `STORAGE_FORCE_PATH_STYLE` - the `*_S3_BUCKET_NAME` variables AWS S3, Cloudflare R2, SeaweedFS and other S3-compatible stores can be used with their own values. Sandbox storage mounts additionally require -`SERVICE_AWS_STORAGE_ROLE_ARN` and an STS-capable store. +`STORAGE_ROLE_ARN` and an STS-capable store. ## Configure shared cache and vector storage diff --git a/packages/storage/README.md b/packages/storage/README.md index 4c60ad0..3836dc1 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -10,12 +10,13 @@ override; nothing in the platform imports it by name. | Variable | Purpose | | ------------------------------- | ---------------------------------------------------------------------- | -| `SERVICE_AWS_REGION` | Region the buckets live in (any non-empty value for stores without regions) | -| `SERVICE_AWS_ACCESS_KEY_ID` | Credentials used for every operation | -| `SERVICE_AWS_SECRET_ACCESS_KEY` | | -| `SERVICE_AWS_ENDPOINT` | S3-compatible endpoint; unset means AWS proper | -| `SERVICE_AWS_FORCE_PATH_STYLE` | `true` for stores without wildcard DNS in front (most self-hosted ones) | -| `SERVICE_AWS_STORAGE_ROLE_ARN` | Role assumed to mint prefix-scoped credentials for sandbox mounts | +| `STORAGE_REGION` | Region the buckets live in (any non-empty value for stores without regions) | +| `STORAGE_ACCESS_KEY_ID` | Credentials used for every operation | +| `STORAGE_SECRET_ACCESS_KEY` | | +| `STORAGE_ENDPOINT` | S3-compatible endpoint; unset means AWS proper | +| `STORAGE_FORCE_PATH_STYLE` | `true` for stores without wildcard DNS in front (most self-hosted ones) | +| `STORAGE_PUBLIC_ENDPOINT` | Endpoint browsers reach for presigned upload/download URLs, when it differs from `STORAGE_ENDPOINT`; unset, presigning uses the server endpoint | +| `STORAGE_ROLE_ARN` | Role assumed to mint prefix-scoped credentials for sandbox mounts | Which bucket backs which logical store is this package's business alone - the platform names a scope, and each scope resolves its bucket from its own @@ -26,7 +27,7 @@ variable: `FILE_S3_BUCKET_NAME`, `IMAGE_S3_BUCKET_NAME`, `VIDEO_S3_BUCKET_NAME`, bucket. Sandbox storage mounts are the one AWS-shaped feature: they mint prefix-scoped -credentials through STS AssumeRole, so they need `SERVICE_AWS_STORAGE_ROLE_ARN` +credentials through STS AssumeRole, so they need `STORAGE_ROLE_ARN` and a store with a compatible STS behind it. Until that is set, everything except sandbox mounts works, and `assertConfigured` fails with a message naming it. diff --git a/packages/storage/src/adapter.test.js b/packages/storage/src/adapter.test.js index 42f070e..373f02f 100644 --- a/packages/storage/src/adapter.test.js +++ b/packages/storage/src/adapter.test.js @@ -22,7 +22,7 @@ class Command { } jest.unstable_mockModule('@aws-sdk/client-s3', () => ({ - S3Client: jest.fn(() => ({ send })), + S3Client: jest.fn((config) => ({ send, config })), ListObjectsV2Command: class extends Command {}, HeadObjectCommand: class extends Command {}, GetObjectCommand: class extends Command {}, @@ -61,9 +61,9 @@ beforeEach(() => { getSignedUrl.mockClear() Object.assign(process.env, { - SERVICE_AWS_REGION: 'eu-west-1', - SERVICE_AWS_ACCESS_KEY_ID: 'key', - SERVICE_AWS_SECRET_ACCESS_KEY: 'secret', + STORAGE_REGION: 'eu-west-1', + STORAGE_ACCESS_KEY_ID: 'key', + STORAGE_SECRET_ACCESS_KEY: 'secret', FILE_S3_BUCKET_NAME: 'files-bucket', SPACE_S3_BUCKET_NAME: 'spaces-bucket', @@ -409,6 +409,81 @@ describe('getObjectUploadUrl', () => { }) }) +describe('public endpoint', () => { + // @note SigV4 signs the host, so a URL a browser will use has to be signed + // against the address the browser reaches - not the one the server does + + afterEach(() => { + delete process.env.STORAGE_ENDPOINT + delete process.env.STORAGE_PUBLIC_ENDPOINT + delete process.env.STORAGE_FORCE_PATH_STYLE + }) + + it('presigns against the public endpoint and sends through the server one', async () => { + Object.assign(process.env, { + STORAGE_ENDPOINT: 'http://garage:3900', + STORAGE_PUBLIC_ENDPOINT: 'http://localhost:3900', + STORAGE_FORCE_PATH_STYLE: 'true', + }) + + const storage = await load() + + send.mockResolvedValue({}) + + await storage.headObject('file', 'k') + await storage.getObjectUploadUrl('file', 'k') + await storage.getObjectDownloadUrl('file', 'k') + + const { S3Client } = await import('@aws-sdk/client-s3') + + expect(S3Client.mock.calls.map(([config]) => config.endpoint)).toEqual([ + 'http://garage:3900', + 'http://localhost:3900', + ]) + + for (const [client] of getSignedUrl.mock.calls) { + expect(client.config).toMatchObject({ + endpoint: 'http://localhost:3900', + forcePathStyle: true, + region: 'eu-west-1', + credentials: { accessKeyId: 'key', secretAccessKey: 'secret' }, + }) + } + }) + + it('presigns against the server endpoint when no public one is set', async () => { + Object.assign(process.env, { STORAGE_ENDPOINT: 'http://garage:3900' }) + + delete process.env.STORAGE_PUBLIC_ENDPOINT + + const storage = await load() + + await storage.getObjectUploadUrl('file', 'k') + + expect(getSignedUrl.mock.calls[0][0].config.endpoint).toBe( + 'http://garage:3900' + ) + }) + + it('never bakes a body checksum into a presigned URL', async () => { + // @note the body is unknown at minting time; a checksum computed then is + // that of an empty body, and a store that honours it rejects the upload + const storage = await load() + + send.mockResolvedValue({}) + + await storage.putObject('file', 'k', 'b') + await storage.getObjectUploadUrl('file', 'k') + + const { S3Client } = await import('@aws-sdk/client-s3') + + expect(S3Client.mock.calls[0][0].requestChecksumCalculation).toBeUndefined() + expect(getSignedUrl.mock.calls[0][0].config.requestChecksumCalculation).toBe( + 'WHEN_REQUIRED' + ) + }) +}) + describe('ephemeralUrlPattern', () => { it('matches a presigned URL', () => { const url = diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index a5aac89..ade69e4 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -3,7 +3,7 @@ // contract's neutral shapes rather than the SDK's, so that callers never // depend on which service is behind them. // -// It speaks the protocol, not the vendor: point SERVICE_AWS_ENDPOINT at any +// It speaks the protocol, not the vendor: point STORAGE_ENDPOINT at any // S3-compatible store (Garage, SeaweedFS, R2, or AWS itself, which is also // the default when no endpoint is set). Sandbox storage mounts are the one // AWS-shaped exception - they mint prefix-scoped credentials through STS @@ -52,15 +52,22 @@ import { getBucketAccessCredentials, getStorageRoleArn } from './sts' export type * from '@chatbotkit-dev/storage-spec' const schema = z.object({ - SERVICE_AWS_REGION: z.string(), - SERVICE_AWS_ACCESS_KEY_ID: z.string(), - SERVICE_AWS_SECRET_ACCESS_KEY: z.string(), + STORAGE_REGION: z.string(), + STORAGE_ACCESS_KEY_ID: z.string(), + STORAGE_SECRET_ACCESS_KEY: z.string(), // @note unset means AWS proper. Any S3-compatible endpoint works here; // self-hosted stores usually need path-style addressing too, since // virtual-host style implies wildcard DNS in front of the store. - SERVICE_AWS_ENDPOINT: z.string().url().optional(), - SERVICE_AWS_FORCE_PATH_STYLE: z.enum(['true', 'false']).optional(), + STORAGE_ENDPOINT: z.string().url().optional(), + STORAGE_FORCE_PATH_STYLE: z.enum(['true', 'false']).optional(), + + // @note where a browser reaches the store when that differs from where the + // server does - a store on a private network, addressed by a name only the + // server resolves. SigV4 signs the host, so presigned URLs must be minted + // against the address the browser will use. Unset, presigning uses the + // server endpoint. + STORAGE_PUBLIC_ENDPOINT: z.string().url().optional(), }) // @note which bucket backs which logical store is this package's business @@ -186,15 +193,15 @@ function getClient(): S3Client { const env = getEnv() cachedClient = new S3Client({ - region: env.SERVICE_AWS_REGION, + region: env.STORAGE_REGION, - ...(env.SERVICE_AWS_ENDPOINT && { endpoint: env.SERVICE_AWS_ENDPOINT }), + ...(env.STORAGE_ENDPOINT && { endpoint: env.STORAGE_ENDPOINT }), - forcePathStyle: env.SERVICE_AWS_FORCE_PATH_STYLE === 'true', + forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === 'true', credentials: { - accessKeyId: env.SERVICE_AWS_ACCESS_KEY_ID, - secretAccessKey: env.SERVICE_AWS_SECRET_ACCESS_KEY, + accessKeyId: env.STORAGE_ACCESS_KEY_ID, + secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY, }, }) } @@ -202,6 +209,36 @@ function getClient(): S3Client { return cachedClient } +let cachedPresignClient: S3Client | undefined + +function getPresignClient(): S3Client { + if (!cachedPresignClient) { + const env = getEnv() + + const endpoint = env.STORAGE_PUBLIC_ENDPOINT ?? env.STORAGE_ENDPOINT + + cachedPresignClient = new S3Client({ + region: env.STORAGE_REGION, + + ...(endpoint && { endpoint }), + + forcePathStyle: env.STORAGE_FORCE_PATH_STYLE === 'true', + + credentials: { + accessKeyId: env.STORAGE_ACCESS_KEY_ID, + secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY, + }, + + // @note the body is unknown when a URL is minted, so a default checksum + // would be that of an empty body, baked into the URL - and a store that + // honours it rejects the real upload + requestChecksumCalculation: 'WHEN_REQUIRED', + }) + } + + return cachedPresignClient +} + /** * The SDK's `Body` is a stream augmented with transform helpers, and which * helpers exist depends on the runtime. This narrows it to the three the @@ -549,7 +586,7 @@ export async function getObjectDownloadUrl( }), }) - return await getSignedUrl(getClient(), command, { + return await getSignedUrl(getPresignClient(), command, { expiresIn: options?.expiresIn || ONE_DAY_IN_SECONDS, }) } @@ -579,7 +616,7 @@ export async function getObjectUploadUrl( ...(options?.metadata && { Metadata: options.metadata }), }) - return await getSignedUrl(getClient(), command, { + return await getSignedUrl(getPresignClient(), command, { expiresIn: options?.expiresIn || ONE_DAY_IN_SECONDS, }) } @@ -630,10 +667,10 @@ export async function getMounts( const env = getEnv() - const region = env.SERVICE_AWS_REGION + const region = env.STORAGE_REGION return { - endpoint: env.SERVICE_AWS_ENDPOINT ?? `https://s3.${region}.amazonaws.com`, + endpoint: env.STORAGE_ENDPOINT ?? `https://s3.${region}.amazonaws.com`, region, credentials: { @@ -672,7 +709,7 @@ export async function assertConfigured(): Promise { if (!getStorageRoleArn()) { throw new Error( - 'SERVICE_AWS_STORAGE_ROLE_ARN is not set, so sandboxes cannot mount ' + + 'STORAGE_ROLE_ARN is not set, so sandboxes cannot mount ' + 'storage. It replaces a role ARN that used to be hardcoded in the ' + 'platform source, so it must be supplied by the environment now.' ) diff --git a/packages/storage/src/mounts.test.js b/packages/storage/src/mounts.test.js index e7385eb..b8b2955 100644 --- a/packages/storage/src/mounts.test.js +++ b/packages/storage/src/mounts.test.js @@ -56,10 +56,10 @@ async function load(overrides = {}) { } Object.assign(process.env, { - SERVICE_AWS_REGION: 'eu-west-1', - SERVICE_AWS_ACCESS_KEY_ID: 'key', - SERVICE_AWS_SECRET_ACCESS_KEY: 'secret', - SERVICE_AWS_STORAGE_ROLE_ARN: 'arn:aws:iam::123:role/storage', + STORAGE_REGION: 'eu-west-1', + STORAGE_ACCESS_KEY_ID: 'key', + STORAGE_SECRET_ACCESS_KEY: 'secret', + STORAGE_ROLE_ARN: 'arn:aws:iam::123:role/storage', ...SCOPES, ...overrides, }) @@ -141,14 +141,14 @@ describe('getMounts', () => { it('says what to set when the role is not configured', async () => { const { getMounts } = await load({ - SERVICE_AWS_STORAGE_ROLE_ARN: undefined, + STORAGE_ROLE_ARN: undefined, }) // @note deliberately an error, not null. Null means the backend cannot mint // scoped credentials at all and the caller degrades past it; this backend // can, and is simply not configured to. await expect(getMounts([{ scope: 'space', prefix: 'p' }])).rejects.toThrow( - /SERVICE_AWS_STORAGE_ROLE_ARN is not set/ + /STORAGE_ROLE_ARN is not set/ ) }) }) @@ -176,11 +176,11 @@ describe('assertConfigured', () => { it('requires the mount role', async () => { const { assertConfigured } = await load({ - SERVICE_AWS_STORAGE_ROLE_ARN: undefined, + STORAGE_ROLE_ARN: undefined, }) await expect(assertConfigured()).rejects.toThrow( - /SERVICE_AWS_STORAGE_ROLE_ARN is not set/ + /STORAGE_ROLE_ARN is not set/ ) }) diff --git a/packages/storage/src/polyfill.test.js b/packages/storage/src/polyfill.test.js index 1a9e8ff..65d9101 100644 --- a/packages/storage/src/polyfill.test.js +++ b/packages/storage/src/polyfill.test.js @@ -38,9 +38,9 @@ jest.unstable_mockModule('@aws-sdk/client-sts', () => ({ })) Object.assign(process.env, { - SERVICE_AWS_REGION: 'eu-west-1', - SERVICE_AWS_ACCESS_KEY_ID: 'key', - SERVICE_AWS_SECRET_ACCESS_KEY: 'secret', + STORAGE_REGION: 'eu-west-1', + STORAGE_ACCESS_KEY_ID: 'key', + STORAGE_SECRET_ACCESS_KEY: 'secret', }) const absentBefore = typeof globalThis.FileReader === 'undefined' diff --git a/packages/storage/src/sts.ts b/packages/storage/src/sts.ts index f94e71c..8d7afd7 100644 --- a/packages/storage/src/sts.ts +++ b/packages/storage/src/sts.ts @@ -12,14 +12,14 @@ import { import { z } from 'zod' const schema = z.object({ - SERVICE_AWS_REGION: z.string(), - SERVICE_AWS_ACCESS_KEY_ID: z.string(), - SERVICE_AWS_SECRET_ACCESS_KEY: z.string(), + STORAGE_REGION: z.string(), + STORAGE_ACCESS_KEY_ID: z.string(), + STORAGE_SECRET_ACCESS_KEY: z.string(), // @note the role the scoped session is assumed into. It was hardcoded in the // platform's source as a literal ARN carrying this deployment's AWS account // number, which is exactly the kind of thing that must not ship in open code. - SERVICE_AWS_STORAGE_ROLE_ARN: z.string().optional(), + STORAGE_ROLE_ARN: z.string().optional(), }) let cachedEnv: z.infer | undefined @@ -39,10 +39,10 @@ function getClient(): STSClient { const env = getEnv() cachedClient = new STSClient({ - region: env.SERVICE_AWS_REGION, + region: env.STORAGE_REGION, credentials: { - accessKeyId: env.SERVICE_AWS_ACCESS_KEY_ID, - secretAccessKey: env.SERVICE_AWS_SECRET_ACCESS_KEY, + accessKeyId: env.STORAGE_ACCESS_KEY_ID, + secretAccessKey: env.STORAGE_SECRET_ACCESS_KEY, }, }) } @@ -54,7 +54,7 @@ function getClient(): STSClient { * Get bucket access credentials based on the given bucket:prefix mapping */ export function getStorageRoleArn(): string | undefined { - return getEnv().SERVICE_AWS_STORAGE_ROLE_ARN + return getEnv().STORAGE_ROLE_ARN } export async function getBucketAccessCredentials( @@ -68,7 +68,7 @@ export async function getBucketAccessCredentials( // it is simply not configured to, and that is a deployment error worth // saying out loud rather than degrading past. throw new Error( - 'SERVICE_AWS_STORAGE_ROLE_ARN is not set, so scoped bucket credentials ' + + 'STORAGE_ROLE_ARN is not set, so scoped bucket credentials ' + 'cannot be issued and sandboxes cannot mount storage. Set it to the ' + 'ARN of the role that grants s3:GetObject/PutObject/DeleteObject on ' + 'the storage buckets.' diff --git a/platform/.env.example b/platform/.env.example index 606df7f..36dd868 100644 --- a/platform/.env.example +++ b/platform/.env.example @@ -47,14 +47,16 @@ PRISMA_DATABASE_URL=file:./.dev/platform.db # refuses storage features at the point of use. The docker compose stack # stands up a provisioned local store (Garage) - these values point at it, so # uncommenting them is all a host-side `pnpm dev` needs. Any S3-compatible -# store (AWS, R2, SeaweedFS) works with its own values; the bucket variables -# and the sandbox-mount role are documented in packages/storage/README.md. -# -# SERVICE_AWS_ENDPOINT=http://localhost:3900 -# SERVICE_AWS_REGION=garage -# SERVICE_AWS_ACCESS_KEY_ID=GK31e57eba9df26b2e7e1b0eaa -# SERVICE_AWS_SECRET_ACCESS_KEY=9f3c1e2b8a4d5f6071829304a5b6c7d8e9f00112233445566778899aabbccdde -# SERVICE_AWS_FORCE_PATH_STYLE=true +# store (AWS, R2, SeaweedFS) works with its own values; the bucket variables, +# STORAGE_PUBLIC_ENDPOINT (where browsers reach the store, when that +# differs from where the server does) and the sandbox-mount role are +# documented in packages/storage/README.md. +# +# STORAGE_ENDPOINT=http://localhost:3900 +# STORAGE_REGION=garage +# STORAGE_ACCESS_KEY_ID=GK31e57eba9df26b2e7e1b0eaa +# STORAGE_SECRET_ACCESS_KEY=9f3c1e2b8a4d5f6071829304a5b6c7d8e9f00112233445566778899aabbccdde +# STORAGE_FORCE_PATH_STYLE=true # FILE_S3_BUCKET_NAME=file # IMAGE_S3_BUCKET_NAME=image # VIDEO_S3_BUCKET_NAME=video diff --git a/platform/config/models.ts b/platform/config/models.ts index f868800..6470a88 100644 --- a/platform/config/models.ts +++ b/platform/config/models.ts @@ -371,6 +371,50 @@ export const openaiLanguageModels: Record< addedDate: '2025-10-06', }, + // GPT-6 + 'gpt-6-astra': { + description: `GPT-6 Astra is OpenAI's most capable model, built for complex reasoning, coding, computer use, research, and document creation across demanding end-to-end workflows.`, + + provider: 'openai', + + family: 'gpt-6', + + features: ['chat', 'functions', 'image', 'reasoning', 'responses'], + + region: 'us', + availableRegions: ['us', 'eu'], + + featured: true, + + maxTokens: 1_050_000, + maxInputTokens: 922_000, + maxOutputTokens: 128_000, + + pricing: { + tokenRatio: 2.7778, + inputTokenRatio: 0.7143, + outputTokenRatio: 2.7778, + inputPrice: 10.0, + outputPrice: 50.0, + }, + + interactionMaxMessages: DEFAULT_INTERACTION_MAX_MESSAGES, + + thresholdStrategy: 'truncate', + + visible: true, + deprecated: false, + + temperature: DEFAULT_TEMPERATURE, + + frequencyPenalty: 0, + presencePenalty: 0, + + tags: [], + + addedDate: '2026-09-04', + }, + // GPT-5 'gpt-5.6-sol': { description: `GPT-5.6 Sol is OpenAI's newest frontier model for the most complex professional work, leading the GPT-5.6 series with advanced reasoning and the strongest coding performance for high-stakes tasks.`, diff --git a/platform/lib/action.exec.file.utest.js b/platform/lib/action.exec.file.utest.js index 1aeb789..31fa290 100644 --- a/platform/lib/action.exec.file.utest.js +++ b/platform/lib/action.exec.file.utest.js @@ -19,9 +19,9 @@ import * as fileAccess from '@/lib/file.access' import * as fileStorage from '@/lib/file.storage' Object.assign(process.env, { - SERVICE_AWS_REGION: 'us-east-1', - SERVICE_AWS_ACCESS_KEY_ID: 'test-key', - SERVICE_AWS_SECRET_ACCESS_KEY: 'test-secret', + STORAGE_REGION: 'us-east-1', + STORAGE_ACCESS_KEY_ID: 'test-key', + STORAGE_SECRET_ACCESS_KEY: 'test-secret', }) jest.mock('@/lib/storage', () => ({ diff --git a/platform/lib/model.provider.openai.adaptor.ts b/platform/lib/model.provider.openai.adaptor.ts index 4a88604..b5484c0 100644 --- a/platform/lib/model.provider.openai.adaptor.ts +++ b/platform/lib/model.provider.openai.adaptor.ts @@ -34,6 +34,12 @@ export function convertTemperature( break } + case isModel(model, [/^gpt-6/]): { + temperature = undefined // @note parameter is not supported + + break + } + case isModel(model, [/^gpt-5/, /^o4-mini/, /^o3/]): { temperature = 1 // @note parameter must be set to 1 diff --git a/platform/lib/model.provider.openai.adaptor.utest.js b/platform/lib/model.provider.openai.adaptor.utest.js index 766ddb8..caf642f 100644 --- a/platform/lib/model.provider.openai.adaptor.utest.js +++ b/platform/lib/model.provider.openai.adaptor.utest.js @@ -60,6 +60,16 @@ describe('openai.adaptor', () => { expect(result).toBe(1) }) + it('should return undefined for gpt-6 models', async () => { + isModel.mockImplementation((model, patterns) => { + return patterns.some((p) => p.test(model)) + }) + + const result = await convertTemperature(0, 'gpt-6-astra') + + expect(result).toBeUndefined() + }) + it('should return 1 for o4-mini models', async () => { isModel.mockImplementation((model, patterns) => { return patterns.some((p) => p.test(model)) diff --git a/platform/next.config.d/bundling.agentos.config.js b/platform/next.config.d/bundling.agentos.config.js new file mode 100644 index 0000000..aee282d --- /dev/null +++ b/platform/next.config.d/bundling.agentos.config.js @@ -0,0 +1,37 @@ +// @ts-check + +// The sandbox module's runtime, AgentOS (@rivet-dev/agentos-core), is the one +// dependency whose files the bundler can neither bundle nor trace, so its +// bundling rules live here rather than in bundling.config.js. + +/** @type {import('next').NextConfig} */ +export default { + // @note spawns its native sidecar and resolves its command packages + // against its own package directory; bundling it strands those as + // relative import() externals. It is ESM-only, so this externalizes it as + // an import() the sandbox module awaits lazily - allowed by name in + // scripts/verify-bundle-modules.js - and there is no commonjs mirror in + // bundling.config.js's webpack hook for it + serverExternalPackages: ['@rivet-dev/agentos-core'], + + // @note the native sidecar and command packages are read from disk at run + // time rather than required, so file tracing cannot see them. Only the + // store directory Node resolves through is globbed: the glob follows + // pnpm's sibling links, so the sidecar meta package carries a flattened + // copy of its platform binary and @agentos-software/common one of each + // command package. Matching the platform and command packages by their own + // store directories ships every binary twice (~500 MB) that nothing links + // to. The runtime package itself is left to the trace, and its native + // sidecar and commands directory are not shipped at all - the core package + // resolves neither at this version + outputFileTracingIncludes: { + '/api/**/*': [ + '../node_modules/.pnpm/@rivet-dev+agentos-sidecar@*/node_modules/@rivet-dev/**/*', + '../node_modules/.pnpm/@agentos-software+common@*/node_modules/@agentos-software/**/*', + ], + '/*': [ + '../node_modules/.pnpm/@rivet-dev+agentos-sidecar@*/node_modules/@rivet-dev/**/*', + '../node_modules/.pnpm/@agentos-software+common@*/node_modules/@agentos-software/**/*', + ], + }, +} diff --git a/platform/next.config.d/bundling.agentos.config.utest.js b/platform/next.config.d/bundling.agentos.config.utest.js new file mode 100644 index 0000000..3cdf2c5 --- /dev/null +++ b/platform/next.config.d/bundling.agentos.config.utest.js @@ -0,0 +1,24 @@ +import config from './bundling.agentos.config' + +describe('bundling.agentos.config', () => { + it('externalizes the sandbox runtime on the server', () => { + expect(config.serverExternalPackages).toEqual(['@rivet-dev/agentos-core']) + }) + + it.each(Object.entries(config.outputFileTracingIncludes))( + 'traces each AgentOS package through one store directory for %s', + (_, patterns) => { + expect(patterns).toEqual([ + '../node_modules/.pnpm/@rivet-dev+agentos-sidecar@*/node_modules/@rivet-dev/**/*', + '../node_modules/.pnpm/@agentos-software+common@*/node_modules/@agentos-software/**/*', + ]) + + // @note a `+name*` or `+*` store glob also matches the platform binary + // and command packages by their own directories, shipping every + // binary a second time; the `@*` anchor pins each glob to one package + for (const pattern of patterns) { + expect(pattern).toMatch(/\.pnpm\/@[^/]+@\*\//) + } + } + ) +}) diff --git a/platform/next.config.d/bundling.config.js b/platform/next.config.d/bundling.config.js index 99c1cbf..bd3b4e1 100644 --- a/platform/next.config.d/bundling.config.js +++ b/platform/next.config.d/bundling.config.js @@ -23,33 +23,16 @@ export default { // @note the egress boundary's dispatcher (lib/egress.ts); left external // so Node's global fetch receives undici's own Agent class 'undici', - // @note spawns its native sidecar and resolves its command packages - // against its own package directory; bundling it strands those as - // relative import() externals - '@rivet-dev/agentos-core', ], // @note include Prisma client files (including WASM) in Vercel serverless functions // @see https://github.com/prisma/prisma/issues/27754 - // @note the sandbox runtime's native sidecar, WebAssembly commands and - // command packages are read from disk at run time rather than required, so - // file tracing cannot see them; the standalone build carries each package - // directory whole, manifest included, so the copies resolve. The runtime - // package itself is left to the trace on purpose: globbing its store - // directory copies its sibling links as flattened files, and the sidecar - // link copied that way can no longer find its platform binary package outputFileTracingIncludes: { '/api/**/*': [ '../../node_modules/.pnpm/@prisma+client*/node_modules/.prisma/client/**/*', - '../node_modules/.pnpm/@rivet-dev+agentos-sidecar*/node_modules/@rivet-dev/**/*', - '../node_modules/.pnpm/@rivet-dev+agentos-runtime-*/node_modules/@rivet-dev/**/*', - '../node_modules/.pnpm/@agentos-software+*/node_modules/@agentos-software/**/*', ], '/*': [ '../../node_modules/.pnpm/@prisma+client*/node_modules/.prisma/client/**/*', - '../node_modules/.pnpm/@rivet-dev+agentos-sidecar*/node_modules/@rivet-dev/**/*', - '../node_modules/.pnpm/@rivet-dev+agentos-runtime-*/node_modules/@rivet-dev/**/*', - '../node_modules/.pnpm/@agentos-software+*/node_modules/@agentos-software/**/*', ], }, @@ -70,9 +53,6 @@ export default { bufferutil: 'commonjs bufferutil', 'utf-8-validate': 'commonjs utf-8-validate', }) - // @note no mirror for @rivet-dev/agentos-core: it is ESM-only, so the - // list above externalizes it as an import() the sandbox module awaits - // lazily - allowed by name in scripts/verify-bundle-modules.js } return config