diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..05384c2db --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Test fixtures are hashed byte-for-byte by the deploy tests; CRLF checkout +# on Windows would change the bytes and break the content-addressed hashes. +packages/cli/tests/fixtures/** text=auto eol=lf diff --git a/bun.lock b/bun.lock index 82ffd1bb8..d7e535e73 100644 --- a/bun.lock +++ b/bun.lock @@ -11,7 +11,7 @@ }, "packages/cli": { "name": "base44", - "version": "0.1.5", + "version": "0.1.7", "bin": { "base44": "./bin/run.js", }, @@ -49,7 +49,7 @@ "express": "^5.0.1", "front-matter": "^4.0.2", "get-port": "^7.1.0", - "globby": "^16.1.0", + "globby": "^16.2.2", "http-proxy-middleware": "^3.0.5", "json-schema-to-typescript": "^15.0.4", "json5": "^2.2.3", @@ -61,6 +61,7 @@ "nanoid": "^5.1.6", "open": "^11.0.0", "outdent": "^0.8.0", + "p-map": "^7.0.6", "p-wait-for": "^6.0.0", "posthog-node": "5.21.2", "qs": "^6.12.3", @@ -658,7 +659,7 @@ "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], - "globby": ["globby@16.1.0", "", { "dependencies": { "@sindresorhus/merge-streams": "^4.0.0", "fast-glob": "^3.3.3", "ignore": "^7.0.5", "is-path-inside": "^4.0.0", "slash": "^5.1.0", "unicorn-magic": "^0.4.0" } }, "sha512-+A4Hq7m7Ze592k9gZRy4gJ27DrXRNnC1vPjxTt1qQxEY8RxagBkBxivkCwg7FxSTG0iLLEMaUx13oOr0R2/qcQ=="], + "globby": ["globby@16.2.2", "", { "dependencies": { "@sindresorhus/merge-streams": "^4.0.0", "fast-glob": "^3.3.3", "ignore": "^7.0.5", "is-path-inside": "^4.0.0", "slash": "^5.1.0", "unicorn-magic": "^0.4.0" } }, "sha512-NLvV9ubZ6NDsJaOpKPy3cQeJpKi9DcWiyCiFUpJPA0YihRqiE6RWaLUmgNNPr8MgPpLZjnBjSmou7uZBRJv9wA=="], "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], @@ -836,6 +837,8 @@ "oxc-resolver": ["oxc-resolver@11.17.1", "", { "optionalDependencies": { "@oxc-resolver/binding-android-arm-eabi": "11.17.1", "@oxc-resolver/binding-android-arm64": "11.17.1", "@oxc-resolver/binding-darwin-arm64": "11.17.1", "@oxc-resolver/binding-darwin-x64": "11.17.1", "@oxc-resolver/binding-freebsd-x64": "11.17.1", "@oxc-resolver/binding-linux-arm-gnueabihf": "11.17.1", "@oxc-resolver/binding-linux-arm-musleabihf": "11.17.1", "@oxc-resolver/binding-linux-arm64-gnu": "11.17.1", "@oxc-resolver/binding-linux-arm64-musl": "11.17.1", "@oxc-resolver/binding-linux-ppc64-gnu": "11.17.1", "@oxc-resolver/binding-linux-riscv64-gnu": "11.17.1", "@oxc-resolver/binding-linux-riscv64-musl": "11.17.1", "@oxc-resolver/binding-linux-s390x-gnu": "11.17.1", "@oxc-resolver/binding-linux-x64-gnu": "11.17.1", "@oxc-resolver/binding-linux-x64-musl": "11.17.1", "@oxc-resolver/binding-openharmony-arm64": "11.17.1", "@oxc-resolver/binding-wasm32-wasi": "11.17.1", "@oxc-resolver/binding-win32-arm64-msvc": "11.17.1", "@oxc-resolver/binding-win32-ia32-msvc": "11.17.1", "@oxc-resolver/binding-win32-x64-msvc": "11.17.1" } }, "sha512-pyRXK9kH81zKlirHufkFhOFBZRks8iAMLwPH8gU7lvKFiuzUH9L8MxDEllazwOb8fjXMcWjY1PMDfMJ2/yh5cw=="], + "p-map": ["p-map@7.0.6", "", {}, "sha512-I4Prw6ivkd6p8PiYR1tXASOAOBzIJwu0TB7fqaX0c/8c3QAehNYmX57EijyGGGBt3c/BIowGwV03RVBtXvHEVg=="], + "p-wait-for": ["p-wait-for@6.0.0", "", {}, "sha512-2kKzMtjS8TVcpCOU/gr3vZ4K/WIyS1AsEFXFWapM/0lERCdyTbB6ZeuCIp+cL1aeLZfQoMdZFCBTHiK4I9UtOw=="], "parse-ms": ["parse-ms@4.0.0", "", {}, "sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw=="], diff --git a/docs/AGENTS.md b/docs/AGENTS.md index 4f04d7f49..453cff780 100644 --- a/docs/AGENTS.md +++ b/docs/AGENTS.md @@ -79,6 +79,7 @@ Read these when working on the relevant area: - **[Adding or modifying CLI commands](commands.md)** - Factory pattern, `runCommand()`, `runTask()`, `CLIContext`, theming, `chalk` ban - **[Making API calls](api-patterns.md)** - HTTP clients, Zod snake_case-to-camelCase transforms, `ApiError.fromHttpError()` - **[Working with resources](resources.md)** - `Resource` interface, adding new resources, site module, unified deploy +- **[Deployments API](deployments.md)** - Static-site deploys addressed by commit, asset manifest hashing, presigned uploads, index.html finalize sentinel - **[Plugins](plugins.md)** - Plugin config, namespaces, entity extension rules, function namespacing, pull/deploy behavior - **[Error handling](error-handling.md)** - Error hierarchy, throwing patterns, error codes, `CLIExitError`, `process.exit` ban - **[Writing tests](testing.md)** - Testkit, Given/When/Then pattern, API mocks, fixtures, test overrides diff --git a/docs/deployments.md b/docs/deployments.md new file mode 100644 index 000000000..d31969689 --- /dev/null +++ b/docs/deployments.md @@ -0,0 +1,49 @@ +# Deployments API (Static Sites) + +**Keywords:** deployments, static site, asset manifest, hash, git hash, commit, presigned, S3, finalize, index.html sentinel, BASE44_STATIC_DEPLOYMENTS, upload + +Deployments ship an app's built output addressed by the commit that produced it. This is a transport of the site module, not a module of its own, so it lives directly in `src/core/site/`: `manifest.ts` (asset walk + hashing), `static-site.ts` (the flow), `upload.ts` (presigned PUTs), with the requests and responses in the shared `api.ts` / `schema.ts` next to the legacy tar.gz upload. Today it carries the env-gated static-site lane; the create response is an ADT designed so a worker (`cf`) arm can slot in next to the static (`s3`) arm without protocol changes — that is the progressive-upgrade path for full-stack apps. + +**Deploying builds — it never publishes.** A deployment is addressed by the commit that produced the build: the server derives the deployment id from `git_hash`, so one commit means one deployment and re-deploying a commit is idempotent. What production serves is decided by the platform publish flow, not by this CLI — there is no `--prod`, no promote/rollback, and no deployment list/logs surface. + +## The Commit Address + +`--git-hash` carries it, and there is no fallback: the flag is what selects this lane, so a deploy without one is the legacy tar.gz upload. A non-hex value is rejected by the option's `argParser` before the action runs — `isGitCommitHash()` in `src/core/utils/git.ts`, pattern `^[a-fA-F0-9]{7,64}$`, the same validation as the server. + +## API Contract (app-scoped, via `getAppClient()`) + +1. `POST deployments` — JSON body: `git_hash` (required) and `asset_manifest` (`{"/path": {hash, size}}`). The response is `{deployment_id, asset_uploads}` where `deployment_id` is a handle for the rest of the flow and `asset_uploads` says where the assets still owed should go, discriminated on `type`: + - `{type: "s3", uploads: [{path, content_type, content_length, url}]}` — one presigned S3 PUT per asset still to upload, **always excluding `/index.html`** (finalize carries it). + - `null` — nothing owed: no assets, or the build already exists (re-deploying a commit is idempotent). +2. **Asset upload — bytes never pass through the backend.** Each upload's raw file bytes are `PUT` directly to its presigned `url` with the signed `content_type` sent verbatim (the URL also signs `content_length`, so the body must be exactly the declared bytes). The URL itself is the credential, so no auth headers and never the app client. Per file: 3 attempts with exponential backoff. Concurrency defaults to `DEFAULT_UPLOAD_CONCURRENCY` (3) and is overridable with `--concurrency `, capped at `MAX_UPLOAD_CONCURRENCY` (50) because each worker holds a whole file in memory. +3. `POST deployments/{id}/finalize` — multipart with exactly one file field named `index.html` carrying the index.html bytes (contentType `text/html`) — no other fields. index.html is the sentinel that completes the deployment, which is also why it never appears in the uploads. Returns `{deployment_id}`. + +## Asset Manifest & Hashing + +`hash = first 32 hex chars of sha256(utf8(app_id) || raw file bytes)` — see `hashAsset()` in `src/core/site/manifest.ts`. The app-id salt is a cache-poisoning defense: a tenant can only produce hash collisions with its own files. + +The output directory is walked with `globby` (`**/*`, dotfiles included, symlinks not followed). `.assetsignore` at the root is honored via globby's `ignoreFiles`, which parses it with the `ignore` package — the same library wrangler uses — so it gets real gitignore semantics: anchoring, directory patterns, `**`, literal braces/extglobs, and **negation** (`!.dev.vars.example` after `.dev.vars*`). Do not translate the patterns by hand, and do not pass globby's `ignore` option alongside `ignoreFiles`: globby globs for ignore files using that option, so it would then find none and silently apply no patterns at all. `.assetsignore` itself, `wrangler.json`, and `.dev.vars` are dropped from the results by name instead. Files over 25 MiB fail with a per-file error; total file count is capped at 100,000. Manifest keys are `/`-prefixed forward-slash paths. + +Content types are deliberately **not** derived client-side: the server decides each asset's Content-Type, signs it into the presigned URL, and the CLI echoes it verbatim — deriving our own value would 403 on any mapping difference. + +## The Static Lane (experimental, env-gated) + +Two switches, in order. **`BASE44_STATIC_DEPLOYMENTS=1`** (or `true`) is the release gate, read by `staticDeploymentsEnabled()` and consulted in exactly one place — `getSiteDeployCommand()`, which registers `--git-hash` only when it is set. **`--git-hash `** is then the runtime switch: passing it deploys through the deployments API, omitting it takes the legacy tar.gz upload. So with the gate off the flag does not exist and the lane is unreachable; with it on, the flow is chosen per invocation. + +On the lane, the output directory becomes the asset manifest (index.html included — it is only ever excluded from uploads), the CLI PUTs each requested file directly to its presigned URL and finalizes with the index.html bytes; today's serving keeps working because the server stores the result the way the legacy site upload does. `--json` emits `{deploymentId, gitHash}`. + +**The lane is reachable only from `base44 site deploy`** — the command the build sandbox drives, since it ships the site rather than the whole project. The fork is a plain `if` in that command's action: `options.gitHash` present → `deployStaticSite()`, absent → `deploySite()` (the tar.gz upload). There is no transport-abstraction layer between the command and the two flows; each branch owns its spinner labels and its result shape. + +`base44 deploy` is untouched by this lane: it keeps shipping the site through `deployAll()`'s legacy tar.gz step exactly as before, gate on or off. The sandbox runs `base44 site deploy -y --json --git-hash ` with a scoped `apps:deploy` workspace key. When the unified deploy should adopt the lane too, that is a deliberate follow-up. + +## Testing + +`TestAPIServer` mocks: `mockDeploymentCreate` (captures the JSON body in `deploymentCreateRequests`; echoes whatever response shape you pass — `asset_uploads` is `{type: "s3", ...}` or `null`), `mockPresignedUpload(path)` (serves a presigned-style `PUT /presigned{path}` target, captures body/Content-Type/Authorization in `presignedUploadRequests`), `mockDeploymentFinalize` (captures multipart fields in `finalizeRequests`). Fixture: `tests/fixtures/with-site/` (static output dir); specs pass `--git-hash` to select the lane. Manifest and ignore-pattern unit tests live in `tests/core/site-manifest.spec.ts`. + +## Rules (Deployments-Specific) + +- **Never re-derive the asset hash** — always go through `hashAsset()` so the app-id salt stays consistent +- **Never derive an upload's Content-Type client-side** — the server signs it into the presigned URL; echo the signed value verbatim +- **Presigned PUTs carry no auth headers and never use the app client** — the URL itself is the scoped credential +- **`git_hash` is required** — a build with no commit behind it has no address and could never be published +- **Legacy behavior stays identical** when the gate is off — the tar.gz site path must not change, and nothing about the lane (flags, help text, output) may surface diff --git a/docs/resources.md b/docs/resources.md index 95f9fc400..0932813bf 100644 --- a/docs/resources.md +++ b/docs/resources.md @@ -76,18 +76,22 @@ Agent skills are app-scoped instruction snippets shared across the app's agents. ## Site Module (Not a Resource) -The site module at `packages/cli/src/core/site/` handles deploying built frontend files. It follows a different pattern than resources: +The site module at `packages/cli/src/core/site/` handles deploying an app's built output. It follows a different pattern than resources — there is no item list, so no `readAll`/`push`. -- Reads built artifacts (JS, CSS, HTML) from the output directory -- Gets configuration from `site.outputDirectory` in project config -- Creates a tar.gz archive and uploads it via `POST /api/apps/{app_id}/deploy-dist` +It exposes **two ways to ship `site.outputDirectory`**, and the caller picks: ```typescript -import { deploySite } from "@/core/site/index.js"; +import { deploySite, deployStaticSite } from "@/core/site/index.js"; -const { appUrl } = await deploySite("./dist"); +// Legacy: tar.gz the built files, POST /api/apps/{app_id}/deploy-dist +const { appUrl } = await deploySite(outputDir); + +// Deployments API (env-gated lane, see deployments.md) +const { deploymentId } = await deployStaticSite({ outputDir, gitHash }); ``` +`base44 site deploy` chooses between them on whether `--git-hash` was passed; `base44 deploy` always uses `deploySite()` via `deployAll()`. The lane's own files are `gate.ts`, `manifest.ts`, `static-site.ts`, and `upload.ts`; both transports share the module's `api.ts` and `schema.ts`. + ### Deploy Flow 1. Validate output directory exists and has files @@ -116,7 +120,7 @@ What it deploys (in order): 3. Agent skills (via `agentSkillResource.push()`) 4. Agents (via `agentResource.push()`) 5. Connectors (via `pushConnectors()`) -- may return OAuth redirect URLs -6. Site (if `site.outputDirectory` is configured) +6. Site (if `site.outputDirectory` is configured) — the legacy tar.gz upload. The env-gated deployments-API lane is reachable only from `base44 site deploy`, not from here (see [deployments.md](deployments.md)). ```bash base44 deploy # With confirmation prompt diff --git a/docs/testing.md b/docs/testing.md index 3a81485a1..f34052dc6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -298,6 +298,20 @@ t.api.mockFunctionLogs("my-function", [ t.api.mockFunctionLogsError("my-function", { status: 500, body: { error: "Server error" } }); ``` +### Deployment Mocks + +See [deployments.md](deployments.md) for the API contract. Requests are captured for assertions: `t.api.deploymentCreateRequests` (JSON bodies), `t.api.presignedUploadRequests` (raw body, Content-Type, Authorization), and `t.api.finalizeRequests` (parsed multipart fields). + +```typescript +t.api.mockDeploymentCreate({ + deployment_id: "app-1-git-a1b2c3d4e5f6", + // {type: "s3", uploads: [...]} or null (nothing owed) + asset_uploads: { type: "s3", uploads: [{ path, content_type, content_length, url }] }, +}); +t.api.mockPresignedUpload("/main.js"); // serves a presigned-style PUT target +t.api.mockDeploymentFinalize({ deployment_id: "app-1-git-a1b2c3d4e5f6" }); +``` + ### Custom Route Mock For advanced scenarios (e.g. stateful responses across retries): diff --git a/packages/cli/package.json b/packages/cli/package.json index b5d867d9b..aa004a9de 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -67,7 +67,7 @@ "express": "^5.0.1", "front-matter": "^4.0.2", "get-port": "^7.1.0", - "globby": "^16.1.0", + "globby": "^16.2.2", "http-proxy-middleware": "^3.0.5", "json-schema-to-typescript": "^15.0.4", "json5": "^2.2.3", @@ -79,6 +79,7 @@ "nanoid": "^5.1.6", "open": "^11.0.0", "outdent": "^0.8.0", + "p-map": "^7.0.6", "p-wait-for": "^6.0.0", "posthog-node": "5.21.2", "qs": "^6.12.3", diff --git a/packages/cli/src/cli/commands/site/deploy.ts b/packages/cli/src/cli/commands/site/deploy.ts index 03ecc2519..eecdda568 100644 --- a/packages/cli/src/cli/commands/site/deploy.ts +++ b/packages/cli/src/cli/commands/site/deploy.ts @@ -1,30 +1,41 @@ import { resolve } from "node:path"; import { confirm, isCancel } from "@clack/prompts"; import type { Command } from "commander"; +import { InvalidArgumentError, Option } from "commander"; import { maybeBuildBeforeDeploy } from "@/cli/commands/project/site-build.js"; import type { CLIContext, RunCommandResult } from "@/cli/types.js"; -import { Base44Command } from "@/cli/utils/index.js"; +import { Base44Command, theme } from "@/cli/utils/index.js"; import { ConfigNotFoundError, InvalidInputError } from "@/core/errors.js"; import { readProjectConfig } from "@/core/project/index.js"; -import { deploySite } from "@/core/site/index.js"; +import { + DEFAULT_UPLOAD_CONCURRENCY, + deploySite, + deployStaticSite, + MAX_UPLOAD_CONCURRENCY, +} from "@/core/site/index.js"; +import { isGitCommitHash } from "@/core/utils/git.js"; interface DeployOptions { yes?: boolean; build?: boolean; + gitHash?: string; + concurrency?: number; } async function deployAction( ctx: CLIContext, options: DeployOptions, ): Promise { - const { isNonInteractive, runTask } = ctx; + const { isNonInteractive } = ctx; if (isNonInteractive && !options.yes) { throw new InvalidInputError("--yes is required in non-interactive mode"); } const { project } = await readProjectConfig(); - if (!project.site?.outputDirectory) { + const outputDirectory = project.site?.outputDirectory; + + if (!outputDirectory) { throw new ConfigNotFoundError("No site configuration found.", { hints: [ { @@ -35,11 +46,9 @@ async function deployAction( }); } - const outputDir = resolve(project.root, project.site.outputDirectory); - if (!options.yes) { const shouldDeploy = await confirm({ - message: `Deploy site from ${project.site.outputDirectory}?`, + message: `Deploy site from ${outputDirectory}?`, }); if (isCancel(shouldDeploy) || !shouldDeploy) { @@ -49,25 +58,126 @@ async function deployAction( await maybeBuildBeforeDeploy(ctx, project, options.build); - const result = await runTask( + const outputDir = resolve(project.root, outputDirectory); + + // A commit means a deployments-API deploy: a deployment is addressed by the + // commit that produced the build. Without one, ship the legacy tar.gz upload. + const { gitHash, concurrency } = options; + + return gitHash + ? await deployToDeploymentsApi(ctx, outputDir, gitHash, concurrency) + : await deployTarball(ctx, outputDir); +} + +async function deployToDeploymentsApi( + { runTask, log, jsonMode }: CLIContext, + outputDir: string, + gitHash: string, + concurrency?: number, +): Promise { + const progressLines: string[] = []; + + const { deploymentId } = await runTask( + "Deploying site...", + async (updateMessage) => + await deployStaticSite({ + outputDir, + gitHash, + concurrency, + progress: { + onAssets: ({ totalAssets, newAssets }) => { + const line = `Found ${totalAssets} static assets (${newAssets} new)`; + progressLines.push(line); + updateMessage(line); + }, + onAssetUpload: ({ uploadedFiles, totalFiles }) => { + updateMessage(`Uploaded ${uploadedFiles} of ${totalFiles} assets`); + }, + }, + }), + { successMessage: "Site deployed", errorMessage: "Site deploy failed" }, + ); + + for (const line of progressLines) { + log.message(theme.styles.dim(line)); + } + + // A build has no URL of its own: what production serves is decided when the + // app is published from the builder, not by this deploy. + return { + outroMessage: `Deployment ${deploymentId} (commit ${gitHash.slice(0, 12)})`, + stdout: jsonMode + ? `${JSON.stringify({ deploymentId, gitHash }, null, 2)}\n` + : undefined, + }; +} + +async function deployTarball( + { runTask }: CLIContext, + outputDir: string, +): Promise { + const { appUrl } = await runTask( "Creating archive and deploying site...", - async () => { - return await deploySite(outputDir); - }, + async () => await deploySite(outputDir), { successMessage: "Site deployed successfully", errorMessage: "Deployment failed", }, ); - return { outroMessage: `Visit your site at: ${result.appUrl}` }; + return { outroMessage: `Visit your site at: ${appUrl}` }; } export function getSiteDeployCommand(): Command { - return new Base44Command("deploy") + const command = new Base44Command("deploy") .description("Deploy built site files to Base44 hosting") .option("-y, --yes", "Skip confirmation prompt") .option("--build", "Build the site before deploying (skips the prompt)") - .option("--no-build", "Deploy without building (skips the prompt)") - .action(deployAction); + .option("--no-build", "Deploy without building (skips the prompt)"); + + // Only registered on the enabled lane, so with the gate off the flag is + // absent from --help and rejected as an unknown option. + if (staticDeploymentsEnabled()) { + command.addOption( + new Option( + "--git-hash ", + "Commit the build came from — deploys through the deployments API", + ).argParser((value) => { + if (!isGitCommitHash(value)) { + throw new InvalidArgumentError( + "Expected a git commit hash (7-64 hex chars).", + ); + } + return value; + }), + ); + command.addOption( + new Option("--concurrency ", "Parallel asset uploads") + .default(DEFAULT_UPLOAD_CONCURRENCY) + .argParser(parseConcurrency), + ); + } + + return command.action(deployAction); +} + +function parseConcurrency(value: string): number { + const parsed = Number(value); + if ( + !Number.isInteger(parsed) || + parsed < 1 || + parsed > MAX_UPLOAD_CONCURRENCY + ) { + throw new InvalidArgumentError( + `Expected a whole number between 1 and ${MAX_UPLOAD_CONCURRENCY}.`, + ); + } + return parsed; +} + +function staticDeploymentsEnabled( + env: NodeJS.ProcessEnv = process.env, +): boolean { + const value = env.BASE44_STATIC_DEPLOYMENTS; + return value === "1" || value === "true"; } diff --git a/packages/cli/src/core/site/api.ts b/packages/cli/src/core/site/api.ts index 9683316e2..bdffd2291 100644 --- a/packages/cli/src/core/site/api.ts +++ b/packages/cli/src/core/site/api.ts @@ -1,8 +1,17 @@ import type { KyResponse } from "ky"; import { getAppClient } from "@/core/clients/index.js"; import { ApiError, SchemaValidationError } from "@/core/errors.js"; -import type { DeployResponse } from "@/core/site/schema.js"; -import { DeployResponseSchema } from "@/core/site/schema.js"; +import type { + CreateDeploymentRequest, + CreateDeploymentResponse, + DeployResponse, + FinalizeDeploymentResponse, +} from "@/core/site/schema.js"; +import { + CreateDeploymentResponseSchema, + DeployResponseSchema, + FinalizeDeploymentResponseSchema, +} from "@/core/site/schema.js"; import { readFile } from "@/core/utils/fs.js"; /** @@ -40,3 +49,74 @@ export async function uploadSite(archivePath: string): Promise { return result.data; } + +export async function createDeployment( + request: CreateDeploymentRequest, +): Promise { + const appClient = getAppClient(); + + let response: KyResponse; + try { + response = await appClient.post("deployments", { + json: request, + timeout: 120_000, + }); + } catch (error) { + throw await ApiError.fromHttpError(error, "creating deployment"); + } + + const result = CreateDeploymentResponseSchema.safeParse( + await response.json(), + ); + if (!result.success) { + throw new SchemaValidationError( + "Invalid response from server", + result.error, + ); + } + return result.data; +} + +/** + * The form carries exactly one file part — `index.html`, the sentinel that + * completes the deployment — and nothing else. + */ +export async function finalizeStaticDeployment( + deploymentId: string, + indexHtml: Uint8Array, +): Promise { + const formData = new FormData(); + formData.append( + "index.html", + new File([indexHtml], "index.html", { type: "text/html" }), + ); + return await postFinalize(deploymentId, formData); +} + +async function postFinalize( + deploymentId: string, + formData: FormData, +): Promise { + const appClient = getAppClient(); + + let response: KyResponse; + try { + response = await appClient.post( + `deployments/${encodeURIComponent(deploymentId)}/finalize`, + { body: formData, timeout: 180_000 }, + ); + } catch (error) { + throw await ApiError.fromHttpError(error, "finalizing deployment"); + } + + const result = FinalizeDeploymentResponseSchema.safeParse( + await response.json(), + ); + if (!result.success) { + throw new SchemaValidationError( + "Invalid response from server", + result.error, + ); + } + return result.data; +} diff --git a/packages/cli/src/core/site/index.ts b/packages/cli/src/core/site/index.ts index 676ceabda..523f1faf8 100644 --- a/packages/cli/src/core/site/index.ts +++ b/packages/cli/src/core/site/index.ts @@ -1,4 +1,7 @@ export * from "./api.js"; export * from "./config.js"; export * from "./deploy.js"; +export * from "./manifest.js"; export * from "./schema.js"; +export * from "./static-site.js"; +export * from "./upload.js"; diff --git a/packages/cli/src/core/site/manifest.ts b/packages/cli/src/core/site/manifest.ts new file mode 100644 index 000000000..4df018841 --- /dev/null +++ b/packages/cli/src/core/site/manifest.ts @@ -0,0 +1,90 @@ +import { createHash } from "node:crypto"; +import { readFile, stat } from "node:fs/promises"; +import { basename, join } from "node:path"; +import { globby } from "globby"; +import { InvalidInputError } from "@/core/errors.js"; +import type { + AssetFile, + AssetManifestEntry, + AssetManifestResult, +} from "./schema.js"; + +const MAX_ASSET_SIZE_BYTES = 25 * 1024 * 1024; // 25 MiB +const MAX_ASSET_COUNT = 100_000; + +const ASSETS_IGNORE_FILE = ".assetsignore"; + +/** Files never uploaded as assets, regardless of .assetsignore. */ +const ALWAYS_IGNORED = new Set([ + ASSETS_IGNORE_FILE, + "wrangler.json", + ".dev.vars", +]); + +/** + * First 32 hex chars of sha256(utf8(app_id) || raw file bytes). The app-id salt + * means a tenant can only collide with their own files, so a malicious upload + * cannot poison another app's asset cache. + */ +export function hashAsset(appId: string, content: Buffer): string { + return createHash("sha256") + .update(Buffer.from(appId, "utf8")) + .update(content) + .digest("hex") + .slice(0, 32); +} + +/** + * Walk the assets directory and build the deployment asset manifest. Honors + * `.assetsignore` at the assets root with full gitignore semantics, negation + * included. + */ +export async function buildAssetManifest( + assetsDir: string, + appId: string, +): Promise { + const manifest: Record = {}; + const filesByHash = new Map(); + + // globby returns forward-slash paths on every platform, which is how the + // manifest keys them. Never pass `ignore` alongside `ignoreFiles`: globby + // globs for ignore files using that option, so it would find none and + // silently apply no patterns — hence the filter below. + const found = await globby("**/*", { + cwd: assetsDir, + dot: true, + onlyFiles: true, + followSymbolicLinks: false, + ignoreFiles: [ASSETS_IGNORE_FILE], + }); + const relativeFilePaths = found.filter( + (path) => !ALWAYS_IGNORED.has(basename(path)), + ); + + if (relativeFilePaths.length > MAX_ASSET_COUNT) { + throw new InvalidInputError( + `Too many static assets: found ${relativeFilePaths.length}, the limit is ${MAX_ASSET_COUNT} files.`, + ); + } + + for (const relativePath of relativeFilePaths.sort()) { + const absolutePath = join(assetsDir, ...relativePath.split("/")); + // Stat before read so an oversized file is never pulled into memory. + const { size } = await stat(absolutePath); + if (size > MAX_ASSET_SIZE_BYTES) { + throw new InvalidInputError( + `Static asset "${relativePath}" is ${size} bytes, which exceeds the 25 MiB per-file limit.`, + ); + } + + const content = await readFile(absolutePath); + const hash = hashAsset(appId, content); + + manifest[`/${relativePath}`] = { hash, size }; + if (!filesByHash.has(hash)) { + filesByHash.set(hash, { absolutePath, hash, size }); + } + } + + return { manifest, filesByHash }; +} diff --git a/packages/cli/src/core/site/schema.ts b/packages/cli/src/core/site/schema.ts index 52afcd312..9897fa2e9 100644 --- a/packages/cli/src/core/site/schema.ts +++ b/packages/cli/src/core/site/schema.ts @@ -16,3 +16,116 @@ export type DeployResponse = z.infer; export const PublishedUrlResponseSchema = z.object({ url: z.string(), }); + +/** Manifest entry keyed by URL-ish path ("/index.html"). */ +export interface AssetManifestEntry { + hash: string; + size: number; +} + +export interface AssetFile { + absolutePath: string; + hash: string; + size: number; +} + +export interface AssetManifestResult { + manifest: Record; + filesByHash: Map; +} + +export interface AssetUploadProgress { + uploadedFiles: number; + totalFiles: number; +} + +export interface DeploymentProgress { + onAssets?: (info: { totalAssets: number; newAssets: number }) => void; + onAssetUpload?: (progress: AssetUploadProgress) => void; +} + +/** + * A request without a worker config is a static-site deployment, which the + * server answers with the `s3` arm of the create response. The deployment id + * is derived from `git_hash`, so re-deploying a commit is idempotent. + */ +export interface CreateDeploymentRequest { + git_hash: string; + asset_manifest: Record; +} + +export interface PresignedAssetUpload { + path: string; + /** Content-Type signed into the URL — the PUT must send it verbatim. */ + contentType: string; + contentLength: number; + /** Presigned S3 URL — the URL itself is the credential. */ + url: string; +} + +interface S3AssetUploads { + type: "s3"; + uploads: PresignedAssetUpload[]; +} + +/** + * `asset_uploads` says where the assets still owed should go, discriminated on + * `type`. The `s3` arm always excludes `/index.html` (finalize carries it), and + * the whole field is null when nothing is owed. + */ +export const CreateDeploymentResponseSchema = z + .object({ + deployment_id: z.string(), + asset_uploads: z + .object({ + type: z.literal("s3"), + uploads: z.array( + z.object({ + path: z.string(), + content_type: z.string(), + content_length: z.number(), + url: z.string(), + }), + ), + }) + .nullable() + .optional(), + }) + .transform( + ( + data, + ): { + deploymentId: string; + assetUploads: S3AssetUploads | null; + } => ({ + deploymentId: data.deployment_id, + assetUploads: + data.asset_uploads == null + ? null + : { + type: "s3", + uploads: data.asset_uploads.uploads.map((upload) => ({ + path: upload.path, + contentType: upload.content_type, + contentLength: upload.content_length, + url: upload.url, + })), + }, + }), + ); + +export type CreateDeploymentResponse = z.infer< + typeof CreateDeploymentResponseSchema +>; + +export const FinalizeDeploymentResponseSchema = z + .object({ + deployment_id: z.string(), + }) + .transform((data) => ({ + deploymentId: data.deployment_id, + })); + +export type FinalizeDeploymentResponse = z.infer< + typeof FinalizeDeploymentResponseSchema +>; diff --git a/packages/cli/src/core/site/static-site.ts b/packages/cli/src/core/site/static-site.ts new file mode 100644 index 000000000..0ab1f3067 --- /dev/null +++ b/packages/cli/src/core/site/static-site.ts @@ -0,0 +1,56 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { InvalidInputError } from "@/core/errors.js"; +import { getAppContext } from "@/core/project/app-config.js"; +import { createDeployment, finalizeStaticDeployment } from "./api.js"; +import { buildAssetManifest } from "./manifest.js"; +import type { DeploymentProgress } from "./schema.js"; +import { uploadPresignedAssets } from "./upload.js"; + +/** + * Deploy a static site build through the deployments API: hash the output + * directory into an asset manifest, create the deployment at the commit's + * address with no worker config, PUT the requested files to their presigned + * URLs, and finalize with the index.html bytes. + */ +export async function deployStaticSite(options: { + outputDir: string; + gitHash: string; + concurrency?: number; + progress?: DeploymentProgress; +}): Promise<{ deploymentId: string }> { + const { outputDir, gitHash, concurrency, progress } = options; + + const assets = await buildAssetManifest(outputDir, getAppContext().id); + // Finalize carries the index.html bytes by contract, so its absence is a + // broken build (or a wrong outputDirectory) — fail before any upload. + if (!assets.manifest["/index.html"]) { + throw new InvalidInputError( + `No index.html found in "${outputDir}" — a static site needs one at the output directory root.`, + ); + } + + const created = await createDeployment({ + git_hash: gitHash, + asset_manifest: assets.manifest, + }); + progress?.onAssets?.({ + totalAssets: Object.keys(assets.manifest).length, + newAssets: created.assetUploads?.uploads.length ?? 0, + }); + + if (created.assetUploads) { + await uploadPresignedAssets(created.assetUploads.uploads, assets, { + concurrency, + onProgress: progress?.onAssetUpload, + }); + } + + const indexHtml = await readFile(join(outputDir, "index.html")); + const finalized = await finalizeStaticDeployment( + created.deploymentId, + new Uint8Array(indexHtml), + ); + + return { deploymentId: finalized.deploymentId }; +} diff --git a/packages/cli/src/core/site/upload.ts b/packages/cli/src/core/site/upload.ts new file mode 100644 index 000000000..e9ccbaa25 --- /dev/null +++ b/packages/cli/src/core/site/upload.ts @@ -0,0 +1,76 @@ +import { readFile } from "node:fs/promises"; +import ky from "ky"; +import pMap from "p-map"; +import { ApiError, InternalError } from "@/core/errors.js"; +import type { + AssetManifestResult, + AssetUploadProgress, + PresignedAssetUpload, +} from "./schema.js"; + +export const DEFAULT_UPLOAD_CONCURRENCY = 3; + +/** Each worker holds a whole file in memory, so the ceiling is a memory bound. */ +export const MAX_UPLOAD_CONCURRENCY = 50; + +const MAX_UPLOAD_ATTEMPTS = 3; +const RETRY_BASE_DELAY_MS = 500; + +/** + * PUT static assets directly to their presigned S3 URLs. A presigned URL + * carries its own authorization in the query string, so each request is a plain + * fetch — never the app client, never an Authorization header. + */ +export async function uploadPresignedAssets( + uploads: PresignedAssetUpload[], + assets: AssetManifestResult, + options: { + concurrency?: number; + onProgress?: (progress: AssetUploadProgress) => void; + } = {}, +): Promise { + const { concurrency = DEFAULT_UPLOAD_CONCURRENCY, onProgress } = options; + let uploadedFiles = 0; + + await pMap( + uploads, + async (upload) => { + await uploadPresignedAsset(upload, assets); + uploadedFiles++; + onProgress?.({ uploadedFiles, totalFiles: uploads.length }); + }, + { concurrency }, + ); +} + +async function uploadPresignedAsset( + upload: PresignedAssetUpload, + assets: AssetManifestResult, +): Promise { + const entry = assets.manifest[upload.path]; + const file = entry && assets.filesByHash.get(entry.hash); + if (!file) { + throw new InternalError( + `Server requested upload of unknown asset path: ${upload.path}`, + ); + } + const content = await readFile(file.absolutePath); + + try { + await ky.put(upload.url, { + body: new Uint8Array(content), + // The server signed this exact Content-Type into the URL — deriving + // our own value would 403 on any mapping difference. + headers: { "Content-Type": upload.contentType }, + timeout: 120_000, + // ky retries network errors and 408/429/5xx only, so a 403 from an + // expired URL fails fast instead of burning every attempt. + retry: { + limit: MAX_UPLOAD_ATTEMPTS - 1, + delay: (attempt) => RETRY_BASE_DELAY_MS * 2 ** (attempt - 1), + }, + }); + } catch (error) { + throw await ApiError.fromHttpError(error, "uploading static assets"); + } +} diff --git a/packages/cli/src/core/utils/git.ts b/packages/cli/src/core/utils/git.ts new file mode 100644 index 000000000..f3a5be13b --- /dev/null +++ b/packages/cli/src/core/utils/git.ts @@ -0,0 +1,6 @@ +/** Abbreviated or full commit hash — the same pattern the server validates. */ +const GIT_HASH_PATTERN = /^[a-fA-F0-9]{7,64}$/; + +export function isGitCommitHash(value: string): boolean { + return GIT_HASH_PATTERN.test(value); +} diff --git a/packages/cli/src/core/utils/index.ts b/packages/cli/src/core/utils/index.ts index 43158af3c..f7a8c1419 100644 --- a/packages/cli/src/core/utils/index.ts +++ b/packages/cli/src/core/utils/index.ts @@ -1,3 +1,4 @@ export * from "./dependencies.js"; export * from "./env.js"; export * from "./fs.js"; +export * from "./git.js"; diff --git a/packages/cli/tests/cli/static_site_deployments.spec.ts b/packages/cli/tests/cli/static_site_deployments.spec.ts new file mode 100644 index 000000000..4d892e089 --- /dev/null +++ b/packages/cli/tests/cli/static_site_deployments.spec.ts @@ -0,0 +1,245 @@ +import { readFileSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { fixture, setupCLITests } from "./testkit/index.js"; + +/** The commit the fixture "build" came from. */ +const GIT_HASH = "0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c"; +const DEPLOYMENT_ID = "test-app-git-0f1e2d3c4b5a"; + +/** Server-side content types differ from the CLI's own mapping on purpose — + * the tests prove the signed value wins. */ +const SIGNED_CONTENT_TYPES: Record = { + "/main.js": "application/javascript", + "/styles.css": "text/css", +}; + +/** Byte counts the server signs into the URLs (from the real fixture files). */ +const FIXTURE_SIZES: Record = Object.fromEntries( + ["/main.js", "/styles.css"].map((path) => [ + path, + readFileSync(join(fixture("with-site"), "site-output", path.slice(1))) + .length, + ]), +); + +interface CreateBody { + git_hash: string; + asset_manifest: Record; +} + +describe("site deploy command (static site through the deployments API, env-gated)", () => { + const t = setupCLITests(); + + /** The s3 create arm: presigned PUT targets for the requested paths. */ + function mockStaticCreate(uploadPaths: string[]) { + t.api.mockDeploymentCreate({ + deployment_id: DEPLOYMENT_ID, + asset_uploads: + uploadPaths.length === 0 + ? null + : { + type: "s3" as const, + uploads: uploadPaths.map((path) => ({ + path, + // Deliberately not what the CLI would derive: the signed value wins. + content_type: `${SIGNED_CONTENT_TYPES[path]}; charset=utf-8`, + content_length: FIXTURE_SIZES[path], + url: `${t.api.baseUrl}/presigned${path}`, + })), + }, + }); + for (const path of uploadPaths) { + t.api.mockPresignedUpload(path); + } + } + + async function readSiteFile(name: string): Promise { + return await readFile(join(fixture("with-site"), "site-output", name)); + } + + it("keeps --git-hash out of the help while the gate is off", async () => { + const siteDeployHelp = await t.run("site", "deploy", "--help"); + + t.expectResult(siteDeployHelp).toSucceed(); + t.expectResult(siteDeployHelp).toContain("--build"); + t.expectResult(siteDeployHelp).toNotContain("--git-hash"); + }); + + it("rejects --git-hash outright while the gate is off", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + + const result = await t.run("site", "deploy", "-y", "--git-hash", GIT_HASH); + + t.expectResult(result).toFail(); + t.expectResult(result).toContain("unknown option"); + expect(t.api.deploymentCreateRequests).toHaveLength(0); + }); + + it("shows --git-hash on site deploy once the gate is on", async () => { + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "1" }); + + const result = await t.run("site", "deploy", "--help"); + + t.expectResult(result).toSucceed(); + t.expectResult(result).toContain("--git-hash"); + }); + + it("keeps the legacy tar.gz site upload when the gate is off", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.api.mockSiteDeploy({ app_url: "https://legacy.example.com" }); + + const result = await t.run("site", "deploy", "-y"); + + t.expectResult(result).toSucceed(); + t.expectResult(result).toContain("https://legacy.example.com"); + expect(t.api.deploymentCreateRequests).toHaveLength(0); + }); + + it("deploys the site output through the deployments API when gated on", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "1" }); + mockStaticCreate(["/main.js", "/styles.css"]); + t.api.mockDeploymentFinalize({ deployment_id: DEPLOYMENT_ID }); + + const result = await t.run("site", "deploy", "-y", "--git-hash", GIT_HASH); + + t.expectResult(result).toSucceed(); + t.expectResult(result).toContain("Found 3 static assets (2 new)"); + t.expectResult(result).toContain("Site deployed"); + t.expectResult(result).toContain(DEPLOYMENT_ID); + + expect(t.api.deploymentCreateRequests).toHaveLength(1); + const body = t.api.deploymentCreateRequests[0] as CreateBody; + expect(body.git_hash).toBe(GIT_HASH); + expect(body).not.toHaveProperty("config"); + expect(Object.keys(body.asset_manifest).sort()).toEqual([ + "/index.html", + "/main.js", + "/styles.css", + ]); + + expect(t.api.presignedUploadRequests).toHaveLength(2); + const byPath = new Map( + t.api.presignedUploadRequests.map((r) => [r.path, r]), + ); + const mainJs = byPath.get("/main.js"); + expect(mainJs?.data.equals(await readSiteFile("main.js"))).toBe(true); + expect(mainJs?.contentType).toBe("application/javascript; charset=utf-8"); + expect(mainJs?.authorization).toBeUndefined(); + const styles = byPath.get("/styles.css"); + expect(styles?.data.equals(await readSiteFile("styles.css"))).toBe(true); + expect(styles?.contentType).toBe("text/css; charset=utf-8"); + expect(styles?.authorization).toBeUndefined(); + + expect(t.api.finalizeRequests).toHaveLength(1); + const fields = t.api.finalizeRequests[0]; + expect(fields.map((f) => f.name)).toEqual(["index.html"]); + expect(fields[0].data.equals(await readSiteFile("index.html"))).toBe(true); + // Bun's compiled binary normalizes Blob types to include the charset. + expect(fields[0].contentType).toMatch(/^text\/html(;\s*charset=utf-8)?$/i); + }); + + it("sends no PUTs and still finalizes when every asset is already stored", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "true" }); + mockStaticCreate([]); + t.api.mockDeploymentFinalize({ deployment_id: DEPLOYMENT_ID }); + + const result = await t.run("site", "deploy", "-y", "--git-hash", GIT_HASH); + + t.expectResult(result).toSucceed(); + t.expectResult(result).toContain("Found 3 static assets (0 new)"); + expect(t.api.presignedUploadRequests).toHaveLength(0); + expect(t.api.finalizeRequests).toHaveLength(1); + expect(t.api.finalizeRequests[0].map((f) => f.name)).toEqual([ + "index.html", + ]); + }); + + it("emits a single JSON document with --json", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "1" }); + mockStaticCreate(["/main.js", "/styles.css"]); + t.api.mockDeploymentFinalize({ deployment_id: DEPLOYMENT_ID }); + + const result = await t.run( + "site", + "deploy", + "-y", + "--git-hash", + GIT_HASH, + "--json", + ); + + t.expectResult(result).toSucceed(); + expect(JSON.parse(result.stdout)).toEqual({ + deploymentId: DEPLOYMENT_ID, + gitHash: GIT_HASH, + }); + }); + + it("takes the legacy path when the gate is on but no commit is passed", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "1" }); + t.api.mockSiteDeploy({ app_url: "https://legacy.example.com" }); + + const result = await t.run("site", "deploy", "-y"); + + t.expectResult(result).toSucceed(); + t.expectResult(result).toContain("https://legacy.example.com"); + expect(t.api.deploymentCreateRequests).toHaveLength(0); + }); + + it("rejects a --git-hash that is not a commit hash", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "1" }); + + const result = await t.run("site", "deploy", "-y", "--git-hash", "nope"); + + t.expectResult(result).toFail(); + t.expectResult(result).toContain("Expected a git commit hash"); + expect(t.api.deploymentCreateRequests).toHaveLength(0); + }); + + it("uploads every asset under a --concurrency override", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "1" }); + mockStaticCreate(["/main.js", "/styles.css"]); + t.api.mockDeploymentFinalize({ deployment_id: DEPLOYMENT_ID }); + + const result = await t.run( + "site", + "deploy", + "-y", + "--git-hash", + GIT_HASH, + "--concurrency", + "1", + ); + + t.expectResult(result).toSucceed(); + expect(t.api.presignedUploadRequests).toHaveLength(2); + }); + + it("rejects a --concurrency outside the allowed range", async () => { + await t.givenLoggedInWithProject(fixture("with-site")); + t.givenEnv({ BASE44_STATIC_DEPLOYMENTS: "1" }); + + const zero = await t.run("site", "deploy", "-y", "--concurrency", "0"); + const huge = await t.run("site", "deploy", "-y", "--concurrency", "999"); + + t.expectResult(zero).toFail(); + t.expectResult(zero).toContain("between 1 and 50"); + t.expectResult(huge).toFail(); + t.expectResult(huge).toContain("between 1 and 50"); + }); + + it("hides --concurrency while the gate is off", async () => { + const result = await t.run("site", "deploy", "--help"); + + t.expectResult(result).toSucceed(); + t.expectResult(result).toNotContain("--concurrency"); + }); +}); diff --git a/packages/cli/tests/cli/testkit/TestAPIServer.ts b/packages/cli/tests/cli/testkit/TestAPIServer.ts index d210e5e2e..9779f6c10 100644 --- a/packages/cli/tests/cli/testkit/TestAPIServer.ts +++ b/packages/cli/tests/cli/testkit/TestAPIServer.ts @@ -202,6 +202,88 @@ interface CreateAppResponse { name: string; } +// ─── DEPLOYMENTS TYPES ────────────────────────────────────── + +interface DeploymentCreateResponse { + deployment_id: string; + /** Where the assets still owed should go; null/omitted = nothing owed. */ + asset_uploads?: { + type: "s3"; + uploads: Array<{ + path: string; + content_type: string; + content_length: number; + url: string; + }>; + } | null; +} + +interface DeploymentFinalizeResponse { + deployment_id: string; +} + +/** A parsed part of a multipart/form-data request body. */ +interface MultipartField { + name: string; + filename?: string; + contentType?: string; + data: Buffer; +} + +/** + * Minimal multipart/form-data parser for captured raw request bodies + * (the global express.raw middleware buffers multipart bodies as-is). + */ +function parseMultipart( + body: Buffer, + contentTypeHeader: string, +): MultipartField[] { + const boundaryMatch = /boundary=(?:"([^"]+)"|([^;]+))/.exec( + contentTypeHeader, + ); + if (!boundaryMatch) { + throw new Error(`No multipart boundary in: ${contentTypeHeader}`); + } + const boundary = `--${boundaryMatch[1] ?? boundaryMatch[2]}`; + + const fields: MultipartField[] = []; + const raw = body.toString("binary"); + const sections = raw.split(boundary).slice(1, -1); // drop preamble + closing "--" + + for (const section of sections) { + const part = section.replace(/^\r\n/, ""); + const headerEnd = part.indexOf("\r\n\r\n"); + if (headerEnd === -1) continue; + + const headerBlock = part.slice(0, headerEnd); + const data = Buffer.from( + part.slice(headerEnd + 4).replace(/\r\n$/, ""), + "binary", + ); + + const nameMatch = /name="([^"]*)"/.exec(headerBlock); + const filenameMatch = /filename="([^"]*)"/.exec(headerBlock); + const typeMatch = /content-type:\s*([^\r\n]+)/i.exec(headerBlock); + + fields.push({ + name: nameMatch?.[1] ?? "", + filename: filenameMatch?.[1], + contentType: typeMatch?.[1].trim(), + data, + }); + } + + return fields; +} + +/** A captured presigned-style asset PUT. */ +interface CapturedPresignedUpload { + path: string; + authorization?: string; + contentType?: string; + data: Buffer; +} + interface ListProjectsResponse { id: string; name: string; @@ -569,6 +651,72 @@ export class TestAPIServer { ); } + // ─── DEPLOYMENT ENDPOINTS ───────────────────────────────── + + /** Captured JSON bodies of POST deployments requests. */ + readonly deploymentCreateRequests: unknown[] = []; + /** Captured presigned-style asset PUTs (see mockPresignedUpload). */ + readonly presignedUploadRequests: CapturedPresignedUpload[] = []; + /** Captured multipart fields of finalize requests. */ + readonly finalizeRequests: MultipartField[][] = []; + + /** + * Mock POST /api/apps/{appId}/deployments. Captures the JSON request body + * in `deploymentCreateRequests`. + */ + mockDeploymentCreate(response: DeploymentCreateResponse): this { + this.pendingRoutes.push({ + method: "POST", + path: `/api/apps/${this.appId}/deployments`, + handler: (req, res) => { + this.deploymentCreateRequests.push(req.body); + res.status(200).json(response); + }, + }); + return this; + } + + /** + * Register a presigned-style PUT target for a static asset: serves + * PUT /presigned{path} — point `asset_uploads[].url` at + * `${baseUrl}/presigned{path}` — capturing the raw body, Content-Type, + * and any Authorization header in `presignedUploadRequests`. + */ + mockPresignedUpload(path: string): this { + this.pendingRoutes.push({ + method: "PUT", + path: `/presigned${path}`, + handler: (req, res) => { + this.presignedUploadRequests.push({ + path, + authorization: req.headers.authorization, + contentType: req.headers["content-type"], + data: req.body as Buffer, + }); + res.status(200).end(); + }, + }); + return this; + } + + /** + * Mock POST /api/apps/{appId}/deployments/{id}/finalize. Captures the + * multipart fields in `finalizeRequests`. + */ + mockDeploymentFinalize(response: DeploymentFinalizeResponse): this { + this.pendingRoutes.push({ + method: "POST", + path: `/api/apps/${this.appId}/deployments/:deploymentId/finalize`, + handler: (req, res) => { + this.finalizeRequests.push( + parseMultipart(req.body as Buffer, req.headers["content-type"] ?? ""), + ); + res.status(200).json(response); + }, + }); + return this; + } + // ─── SECRETS ENDPOINTS ─────────────────────────────────── mockSecretsList(response: SecretsListResponse): this { diff --git a/packages/cli/tests/core/site-manifest.spec.ts b/packages/cli/tests/core/site-manifest.spec.ts new file mode 100644 index 000000000..5974ed2b5 --- /dev/null +++ b/packages/cli/tests/core/site-manifest.spec.ts @@ -0,0 +1,192 @@ +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, truncate, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { buildAssetManifest, hashAsset } from "@/core/site/manifest.js"; + +describe("hashAsset", () => { + it("computes the first 32 hex chars of sha256(utf8(app_id) || bytes)", () => { + expect(hashAsset("test-app-id", Buffer.from("hello world"))).toBe( + "b24ad526981fbac802de45c88c134ba4", + ); + }); + + it("matches a locally computed sha256 over the concatenated bytes", () => { + const expected = createHash("sha256") + .update(Buffer.concat([Buffer.from("app-1"), Buffer.from("content")])) + .digest("hex") + .slice(0, 32); + expect(hashAsset("app-1", Buffer.from("content"))).toBe(expected); + }); + + it("salts with the app id so tenants can only collide with themselves", () => { + const content = Buffer.from("hello world"); + expect(hashAsset("test-app-id", content)).not.toBe( + hashAsset("other-app", content), + ); + }); +}); + +describe("buildAssetManifest", () => { + let assetsDir: string; + + beforeEach(async () => { + assetsDir = await mkdtemp(join(tmpdir(), "b44-assets-")); + }); + + afterEach(async () => { + await rm(assetsDir, { recursive: true, force: true }); + }); + + it("builds manifest keys as /-prefixed forward-slash paths with hash and size", async () => { + await writeFile(join(assetsDir, "index.html"), "

Hello

\n"); + await mkdir(join(assetsDir, "assets")); + await writeFile(join(assetsDir, "assets", "app.js"), "console.log(1);"); + + const { manifest, filesByHash } = await buildAssetManifest( + assetsDir, + "test-app-id", + ); + + expect(Object.keys(manifest).sort()).toEqual([ + "/assets/app.js", + "/index.html", + ]); + expect(manifest["/index.html"]).toEqual({ + hash: hashAsset("test-app-id", Buffer.from("

Hello

\n")), + size: 15, + }); + const entry = manifest["/assets/app.js"]; + expect(filesByHash.get(entry.hash)?.size).toBe(entry.size); + }); + + it("honors .assetsignore patterns (exact names, * globs, directory patterns)", async () => { + await writeFile( + join(assetsDir, ".assetsignore"), + ["secret.txt", "*.log", "private/", "# a comment", ""].join("\n"), + ); + await writeFile(join(assetsDir, "keep.txt"), "keep"); + await writeFile(join(assetsDir, "secret.txt"), "drop"); + await writeFile(join(assetsDir, "debug.log"), "drop"); + await mkdir(join(assetsDir, "private")); + await writeFile(join(assetsDir, "private", "notes.txt"), "drop"); + await mkdir(join(assetsDir, "nested")); + await writeFile(join(assetsDir, "nested", "secret.txt"), "drop"); + await writeFile(join(assetsDir, "nested", "keep.js"), "keep"); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(Object.keys(manifest).sort()).toEqual([ + "/keep.txt", + "/nested/keep.js", + ]); + }); + + it("honors .assetsignore negation", async () => { + await writeFile( + join(assetsDir, ".assetsignore"), + [ + ".dev.vars*", + "!.dev.vars.example", + "secrets/", + "!secrets/public.txt", + ].join("\n"), + ); + await writeFile(join(assetsDir, "index.html"), "hi"); + await writeFile(join(assetsDir, ".dev.vars.local"), "drop"); + await writeFile(join(assetsDir, ".dev.vars.example"), "keep"); + await mkdir(join(assetsDir, "secrets")); + await writeFile(join(assetsDir, "secrets", "key.pem"), "drop"); + await writeFile(join(assetsDir, "secrets", "public.txt"), "drop"); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + // `!secrets/public.txt` does not rescue: git cannot re-include a file + // under an excluded directory. + expect(Object.keys(manifest).sort()).toEqual([ + "/.dev.vars.example", + "/index.html", + ]); + }); + + it("treats braces and extglobs as literal names, like gitignore", async () => { + await writeFile( + join(assetsDir, ".assetsignore"), + ["{foo,bar}.js", "+(baz|qux).js"].join("\n"), + ); + await writeFile(join(assetsDir, "foo.js"), "keep"); + await writeFile(join(assetsDir, "baz.js"), "keep"); + await writeFile(join(assetsDir, "{foo,bar}.js"), "drop"); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(Object.keys(manifest).sort()).toEqual(["/baz.js", "/foo.js"]); + }); + + it("anchors .assetsignore patterns that contain a slash", async () => { + await writeFile( + join(assetsDir, ".assetsignore"), + ["/root-only.txt", "nested/drop.txt", "**/*.map"].join("\n"), + ); + await writeFile(join(assetsDir, "root-only.txt"), "drop"); + await mkdir(join(assetsDir, "nested", "root-only.txt"), { + recursive: true, + }); + await writeFile(join(assetsDir, "nested", "root-only.txt", "keep"), "keep"); + await writeFile(join(assetsDir, "nested", "drop.txt"), "drop"); + await writeFile(join(assetsDir, "nested", "keep.txt"), "keep"); + await writeFile(join(assetsDir, "app.js.map"), "drop"); + await writeFile(join(assetsDir, "nested", "app.js.map"), "drop"); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(Object.keys(manifest).sort()).toEqual([ + "/nested/keep.txt", + "/nested/root-only.txt/keep", + ]); + }); + + it("includes dotfiles that are not ignored", async () => { + await mkdir(join(assetsDir, ".well-known")); + await writeFile(join(assetsDir, ".well-known", "security.txt"), "contact"); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(Object.keys(manifest)).toEqual(["/.well-known/security.txt"]); + }); + + it("always skips .assetsignore, wrangler.json, and .dev.vars", async () => { + await writeFile(join(assetsDir, "index.html"), "hi"); + await writeFile(join(assetsDir, "wrangler.json"), "{}"); + await writeFile(join(assetsDir, ".dev.vars"), "SECRET=1"); + + const { manifest } = await buildAssetManifest(assetsDir, "test-app-id"); + + expect(Object.keys(manifest)).toEqual(["/index.html"]); + }); + + it("rejects files larger than 25 MiB with a per-file error", async () => { + const bigFile = join(assetsDir, "big.bin"); + await writeFile(bigFile, ""); + await truncate(bigFile, 25 * 1024 * 1024 + 1); + + await expect(buildAssetManifest(assetsDir, "test-app-id")).rejects.toThrow( + /"big\.bin".*exceeds the 25 MiB per-file limit/, + ); + }); + + it("dedupes identical files by hash in filesByHash", async () => { + await writeFile(join(assetsDir, "a.txt"), "same"); + await writeFile(join(assetsDir, "b.txt"), "same"); + + const { manifest, filesByHash } = await buildAssetManifest( + assetsDir, + "test-app-id", + ); + + expect(Object.keys(manifest)).toHaveLength(2); + expect(manifest["/a.txt"].hash).toBe(manifest["/b.txt"].hash); + expect(filesByHash.size).toBe(1); + }); +});