From c32aea79b3dd374c5446b125eb8423496580cf39 Mon Sep 17 00:00:00 2001 From: "HomePC\\Kevin" Date: Sun, 13 Sep 2026 00:22:09 +0200 Subject: [PATCH 1/6] test: add alpha3 upgrade recovery baseline --- scripts/alpha1-upgrade-fixture.ts | 47 +++++++- scripts/alpha1-upgrade-smoke.ts | 173 +++++++++++++++++++++++------- scripts/alpha3-upgrade-smoke.ts | 8 ++ 3 files changed, 188 insertions(+), 40 deletions(-) create mode 100644 scripts/alpha3-upgrade-smoke.ts diff --git a/scripts/alpha1-upgrade-fixture.ts b/scripts/alpha1-upgrade-fixture.ts index b938ef8..f94defc 100644 --- a/scripts/alpha1-upgrade-fixture.ts +++ b/scripts/alpha1-upgrade-fixture.ts @@ -3,6 +3,10 @@ import migrationJournal from '../web/drizzle/meta/_journal.json' export type Command = (args: string[], timeoutMs?: number) => Promise +export const ALPHA1_MIGRATION_COUNT = 13 +export const ALPHA3_MIGRATION_COUNT = 17 +export const CURRENT_MIGRATION_COUNT = migrationJournal.entries.length + export interface Alpha1UpgradeFixture { readonly runId: string readonly ownerUserId: string @@ -36,6 +40,7 @@ export interface Alpha1UpgradeFixture { readonly hostHttpSettings: Readonly> readonly unsupportedHostSettings: Readonly> readonly advancedConfig: string + readonly managementOrigin?: string } const postgresPasswordFile = '/run/rentnerproxy/postgres/value' @@ -244,6 +249,7 @@ export async function seedAlpha1UpgradeFixture(input: { readonly command: Command readonly upstreamPort: number readonly runId: string + readonly managementOrigin?: string }): Promise { validateRunId(input.runId) if ( @@ -310,6 +316,13 @@ export async function seedAlpha1UpgradeFixture(input: { const caNotBefore = requireString(metadata.caNotBefore, 'CA notBefore') const caNotAfter = requireString(metadata.caNotAfter, 'CA notAfter') const customPermissionKeys = ['proxy_hosts.view', 'redirect_hosts.view'] as const + if (input.managementOrigin !== undefined && input.managementOrigin.length > 2_048) { + throw new Error('Management origin is too long.') + } + const managementOriginSql = + input.managementOrigin === undefined + ? '' + : `,\n('management_origin_v1',${sqlQuote(JSON.stringify({ version: 1, origin: input.managementOrigin }))}::jsonb)` const sql = ` begin; insert into rentnerproxy.roles (id,key,name,description,is_system) @@ -358,7 +371,7 @@ values (${sqlQuote(disabledTlsProxyHostId)},${sqlQuote(advancedConfig)},${sqlQuo insert into rentnerproxy.system_settings (key,value) values ('proxy_runtime_editor_v1',${sqlQuote(JSON.stringify({ version: 1, httpSettings: globalHttpSettings }))}::jsonb), -(${sqlQuote('proxy_runtime_host_v1:' + disabledTlsProxyHostId)},${sqlQuote(JSON.stringify({ version: 1, httpSettings: hostHttpSettings }))}::jsonb) +(${sqlQuote('proxy_runtime_host_v1:' + disabledTlsProxyHostId)},${sqlQuote(JSON.stringify({ version: 1, httpSettings: hostHttpSettings }))}::jsonb)${managementOriginSql} on conflict (key) do update set value=excluded.value,updated_at=now(); commit; ` @@ -397,6 +410,9 @@ commit; hostHttpSettings, unsupportedHostSettings, advancedConfig, + ...(input.managementOrigin === undefined + ? {} + : { managementOrigin: input.managementOrigin }), } } @@ -413,8 +429,14 @@ export async function assertAlpha1UpgradeFixture(input: { readonly command: Command readonly fixture: Alpha1UpgradeFixture readonly expectAlpha2: boolean + readonly expectedMigrationCount?: number + readonly expectCurrentSchema?: boolean }): Promise { const f = input.fixture + const managementOrigin = + f.managementOrigin === undefined + ? '0' + : `(select count(*) from rentnerproxy.system_settings where key='management_origin_v1' and value=${sqlQuote(JSON.stringify({ version: 1, origin: f.managementOrigin }))}::jsonb)` const base = await queryJson( input.command, input.containerId, @@ -451,12 +473,15 @@ export async function assertAlpha1UpgradeFixture(input: { 'globalSettings', (select count(*) from rentnerproxy.system_settings where key='proxy_runtime_editor_v1' and value=${sqlQuote(JSON.stringify({ version: 1, httpSettings: f.globalHttpSettings }))}::jsonb), 'hostSettings', (select count(*) from rentnerproxy.system_settings where key=${sqlQuote('proxy_runtime_host_v1:' + f.disabledTlsProxyHostId)} and value=${sqlQuote(JSON.stringify({ version: 1, httpSettings: f.hostHttpSettings }))}::jsonb), 'legacy', (select count(*) from rentnerproxy.proxy_host_legacy_settings where proxy_host_id=${sqlQuote(f.disabledTlsProxyHostId)} and advanced_config=${sqlQuote(f.advancedConfig)} and unsupported_settings=${sqlQuote(JSON.stringify(f.unsupportedHostSettings))}::jsonb), + 'managementOrigin', ${managementOrigin}, 'migrations', (select count(*) from drizzle.__drizzle_migrations) ) as value`, ) const number = (key: string) => Number(base[key] ?? 0) - const expectedMigrationCount = input.expectAlpha2 ? migrationJournal.entries.length : 13 + const expectedMigrationCount = + input.expectedMigrationCount ?? + (input.expectAlpha2 ? CURRENT_MIGRATION_COUNT : ALPHA1_MIGRATION_COUNT) if (number('migrations') !== expectedMigrationCount) { throw new Error('Unexpected migration journal state for Alpha 1 fixture.') } @@ -496,6 +521,9 @@ export async function assertAlpha1UpgradeFixture(input: { if (number('globalSettings') !== 1 || number('hostSettings') !== 1) { throw new Error('Alpha 1 fixture HTTP settings were not preserved.') } + if (number('managementOrigin') !== (f.managementOrigin === undefined ? 0 : 1)) { + throw new Error('The legacy management origin was not preserved.') + } if (!input.expectAlpha2) return const migrated = await queryJson( @@ -511,7 +539,12 @@ export async function assertAlpha1UpgradeFixture(input: { 'liveDomains', (select count(*) from rentnerproxy.host_domains where proxy_host_id=${sqlQuote(f.liveProxyHostId)} and domain in (${sqlQuote(f.hostDomain)},${sqlQuote(f.aliasDomain)})), 'redirectDomains', (select count(*) from rentnerproxy.host_domains where redirect_host_id=${sqlQuote(f.redirectHostId)} and domain=${sqlQuote(f.redirectDomain)}), 'policyColumn', exists(select 1 from information_schema.columns where table_schema='rentnerproxy' and table_name='proxy_hosts' and column_name='access_policy_id'), - 'policyNull', (select count(*) from rentnerproxy.proxy_hosts where id in (${sqlQuote(f.liveProxyHostId)},${sqlQuote(f.disabledTlsProxyHostId)}) and access_policy_id is null) + 'policyNull', (select count(*) from rentnerproxy.proxy_hosts where id in (${sqlQuote(f.liveProxyHostId)},${sqlQuote(f.disabledTlsProxyHostId)}) and access_policy_id is null), + 'durableEventCursor', to_regclass('rentnerproxy.certificate_event_cursor') is not null, + 'durableEventReceipts', to_regclass('rentnerproxy.certificate_event_receipts') is not null, + 'currentOperationColumn', exists(select 1 from information_schema.columns where table_schema='rentnerproxy' and table_name='certificates' and column_name='current_operation'), + 'candidateColumn', exists(select 1 from information_schema.columns where table_schema='rentnerproxy' and table_name='certificates' and column_name='candidate'), + 'nextAttemptColumn', exists(select 1 from information_schema.columns where table_schema='rentnerproxy' and table_name='certificates' and column_name='next_attempt_at') ) as value`, ) if ( @@ -521,11 +554,17 @@ export async function assertAlpha1UpgradeFixture(input: { migrated.ipRules !== true || Number(migrated.newRolePermissions ?? 0) !== 16 || migrated.policyColumn !== true || + ((input.expectCurrentSchema ?? true) && + (migrated.durableEventCursor !== true || + migrated.durableEventReceipts !== true || + migrated.currentOperationColumn !== true || + migrated.candidateColumn !== true || + migrated.nextAttemptColumn !== true)) || Number(migrated.policyNull ?? 0) !== 2 || Number(migrated.legacy ?? 0) !== 1 || Number(migrated.liveDomains ?? 0) !== 2 || Number(migrated.redirectDomains ?? 0) !== 1 ) { - throw new Error('Alpha 1 to Alpha 2 migration did not preserve the fixture schema/data.') + throw new Error('Alpha 1 to Alpha 4 migration did not preserve the fixture schema/data.') } } diff --git a/scripts/alpha1-upgrade-smoke.ts b/scripts/alpha1-upgrade-smoke.ts index 8d3042e..71e9cc2 100644 --- a/scripts/alpha1-upgrade-smoke.ts +++ b/scripts/alpha1-upgrade-smoke.ts @@ -8,6 +8,9 @@ import { join } from 'node:path' import { smokeCompose } from './smoke-resources' import { verifyRestoreRollback } from './restore-rollback-smoke' import { + ALPHA1_MIGRATION_COUNT, + ALPHA3_MIGRATION_COUNT, + CURRENT_MIGRATION_COUNT, assertAlpha1UpgradeFixture, seedAlpha1UpgradeFixture, type Alpha1UpgradeFixture, @@ -15,10 +18,44 @@ import { export const ALPHA1_IMAGE = 'ghcr.io/rentnerkev/rentnerproxy:v1.0.0-alpha.1@sha256:f88edb70a80db7c527e1a963e835593f6998ab4541f810e26cf75ffa63da0d3f' -const ALPHA1_REVISION = 'a147176c6096935dc5d9824f671b5f21d89b636b' +export const ALPHA1_REVISION = 'a147176c6096935dc5d9824f671b5f21d89b636b' +export const ALPHA3_IMAGE = + 'ghcr.io/rentnerkev/rentnerproxy:v1.0.0-alpha.3@sha256:f876c9c59c819cf537617ff256ec247eae9897adf632453117496fbea6a28742' +export const ALPHA3_REVISION = 'a1bb0117828606cd10919871a098eb3a794b912e' + +export interface PublishedUpgradeBaseline { + readonly name: 'Alpha 1' | 'Alpha 3' + readonly image: string + readonly version: string + readonly revision: string + readonly migrationCount: number + readonly targetName: 'Alpha 2' | 'Alpha 4' + readonly directoryName: string +} + +export const ALPHA1_BASELINE: PublishedUpgradeBaseline = { + name: 'Alpha 1', + image: ALPHA1_IMAGE, + version: 'v1.0.0-alpha.1', + revision: ALPHA1_REVISION, + migrationCount: ALPHA1_MIGRATION_COUNT, + targetName: 'Alpha 2', + directoryName: 'alpha1-upgrade', +} + +export const ALPHA3_BASELINE: PublishedUpgradeBaseline = { + name: 'Alpha 3', + image: ALPHA3_IMAGE, + version: 'v1.0.0-alpha.3', + revision: ALPHA3_REVISION, + migrationCount: ALPHA3_MIGRATION_COUNT, + targetName: 'Alpha 4', + directoryName: 'alpha3-upgrade', +} + type Command = (argumentsList: string[], timeoutMs?: number) => Promise -interface UpgradeSmokeOptions { +export interface UpgradeSmokeOptions { readonly imageTag: string readonly temporaryRoot: string readonly upstreamPort: number @@ -47,27 +84,28 @@ async function waitFor( throw new Error('Upgrade smoke timed out: ' + label) } -function composeDocument(image: string): string { +function composeDocument(image: string, includePublicOrigin: boolean): string { + const environment = { + SMTP_FROM: '${SMTP_FROM:?Set SMTP_FROM}', + SMTP_HOST: '${SMTP_HOST:?Set SMTP_HOST}', + SMTP_PASSWORD: '${SMTP_PASSWORD:?Set SMTP_PASSWORD}', + SMTP_PORT: '${SMTP_PORT:-587}', + SMTP_SECURE: '${SMTP_SECURE:-false}', + SMTP_USER: '${SMTP_USER:?Set SMTP_USER}', + ...(includePublicOrigin + ? { + RENTNERPROXY_PUBLIC_ORIGIN: + '${RENTNERPROXY_PUBLIC_ORIGIN:?Set RENTNERPROXY_PUBLIC_ORIGIN}', + } + : {}), + } return smokeCompose( JSON.stringify({ services: { rentnerproxy: { image, extra_hosts: ['host.docker.internal:host-gateway'], - environment: { - ...(image === ALPHA1_IMAGE - ? {} - : { - RENTNERPROXY_PUBLIC_ORIGIN: - '${RENTNERPROXY_PUBLIC_ORIGIN:?Set RENTNERPROXY_PUBLIC_ORIGIN}', - }), - SMTP_FROM: '${SMTP_FROM:?Set SMTP_FROM}', - SMTP_HOST: '${SMTP_HOST:?Set SMTP_HOST}', - SMTP_PASSWORD: '${SMTP_PASSWORD:?Set SMTP_PASSWORD}', - SMTP_PORT: '${SMTP_PORT:-587}', - SMTP_SECURE: '${SMTP_SECURE:-false}', - SMTP_USER: '${SMTP_USER:?Set SMTP_USER}', - }, + environment, volumes: ['data:/var/lib/rentnerproxy', 'postgres-base:/var/lib/postgresql'], }, }, @@ -76,17 +114,25 @@ function composeDocument(image: string): string { ) } -export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise { +export async function verifyPublishedUpgrade( + options: UpgradeSmokeOptions, + baseline: PublishedUpgradeBaseline, +): Promise { const { command, passed } = options const runId = randomUUID().replaceAll('-', '').slice(0, 12) const project = 'rentnerproxy-alpha-upgrade-' + runId const restoreProject = project + '-restore' - const directory = join(options.temporaryRoot, 'alpha1-upgrade') + const directory = join(options.temporaryRoot, baseline.directoryName) + const baselineId = baseline.name.toLowerCase().replaceAll(' ', '') + const includePublicOrigin = Boolean(options.environment.RENTNERPROXY_PUBLIC_ORIGIN?.trim()) await mkdir(directory) - const oldComposeFile = join(directory, 'alpha1.compose.json') - const newComposeFile = join(directory, 'alpha2.compose.json') - await writeFile(oldComposeFile, composeDocument(ALPHA1_IMAGE)) - await writeFile(newComposeFile, composeDocument(options.imageTag)) + const oldComposeFile = join(directory, baselineId + '.compose.json') + const newComposeFile = join( + directory, + baseline.targetName.toLowerCase().replaceAll(' ', '') + '.compose.json', + ) + await writeFile(oldComposeFile, composeDocument(baseline.image, includePublicOrigin)) + await writeFile(newComposeFile, composeDocument(options.imageTag, includePublicOrigin)) const compose = (file: string, name = project) => [ 'docker', 'compose', @@ -125,7 +171,7 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise id, 'bun', '-e', - `await Bun.write('/tmp/alpha1-upgrade-ca.pem',${JSON.stringify(fixture.caPem)})`, + `await Bun.write('/tmp/${baselineId}-upgrade-ca.pem',${JSON.stringify(fixture.caPem)})`, ]) for (const host of [fixture.hostDomain, fixture.aliasDomain]) { await waitFor( @@ -148,6 +194,19 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise ])) === options.trafficMarker, 'HTTP ' + host, ) + const servedFingerprint = await command([ + 'docker', + 'exec', + id, + 'sh', + '-ceu', + `printf '' | openssl s_client -connect 127.0.0.1:8443 -servername ${host} -showcerts 2>/dev/null | openssl x509 -noout -fingerprint -sha256`, + ]) + const fingerprint = /fingerprint=([0-9a-f:]+)/iu.exec(servedFingerprint)?.[1] + assert.equal( + fingerprint ? 'sha256:' + fingerprint.replaceAll(':', '').toLowerCase() : null, + fixture.certificateFingerprint, + ) assert.equal( await command([ 'docker', @@ -162,7 +221,7 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise '--noproxy', '*', '--cacert', - '/tmp/alpha1-upgrade-ca.pem', + '/tmp/' + baselineId + '-upgrade-ca.pem', '--resolve', host + ':8443:127.0.0.1', 'https://' + host + ':8443/upgrade-traffic', @@ -198,7 +257,7 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise ) } try { - await command(['docker', 'pull', '--platform', 'linux/amd64', ALPHA1_IMAGE], 900_000) + await command(['docker', 'pull', '--platform', 'linux/amd64', baseline.image], 900_000) const labels = JSON.parse( await command([ 'docker', @@ -206,11 +265,11 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise 'inspect', '--format', '{{json .Config.Labels}}', - ALPHA1_IMAGE, + baseline.image, ]), ) as Record - assert.equal(labels['org.opencontainers.image.version'], 'v1.0.0-alpha.1') - assert.equal(labels['org.opencontainers.image.revision'], ALPHA1_REVISION) + assert.equal(labels['org.opencontainers.image.version'], baseline.version) + assert.equal(labels['org.opencontainers.image.revision'], baseline.revision) await command([...oldCompose, 'up', '--detach'], 240_000) let id = await containerId(oldCompose) await healthy(id) @@ -219,12 +278,24 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise command, upstreamPort: options.upstreamPort, runId, + ...(options.environment.RENTNERPROXY_PUBLIC_ORIGIN?.trim() + ? { managementOrigin: options.environment.RENTNERPROXY_PUBLIC_ORIGIN.trim() } + : {}), + }) + await assertAlpha1UpgradeFixture({ + containerId: id, + command, + fixture, + expectAlpha2: false, + expectedMigrationCount: baseline.migrationCount, + expectCurrentSchema: false, }) - await assertAlpha1UpgradeFixture({ containerId: id, command, fixture, expectAlpha2: false }) await traffic(id, fixture) const originalRevision = await activeRevision(id) passed( - 'published Alpha 1 image starts with real users, roles, hosts, certificates and desired state', + 'published ' + + baseline.name + + ' image starts with real users, roles, hosts, certificates and desired state', ) const backupRoot = join(directory, 'backups') @@ -247,19 +318,33 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise await command([...nextCompose, 'up', '--detach'], 240_000) id = await containerId(nextCompose) await healthy(id) - await assertAlpha1UpgradeFixture({ containerId: id, command, fixture, expectAlpha2: true }) + await assertAlpha1UpgradeFixture({ + containerId: id, + command, + fixture, + expectAlpha2: true, + expectedMigrationCount: CURRENT_MIGRATION_COUNT, + }) await traffic(id, fixture) assert.equal(await activeRevision(id), originalRevision) passed( - 'in-place Alpha 1 upgrade preserves database records and live HTTP, HTTPS and redirects', + 'in-place ' + + baseline.name + + ' upgrade preserves database records and live HTTP, HTTPS and redirects', ) await command([...nextCompose, 'restart', 'rentnerproxy'], 180_000) await healthy(id) - await assertAlpha1UpgradeFixture({ containerId: id, command, fixture, expectAlpha2: true }) + await assertAlpha1UpgradeFixture({ + containerId: id, + command, + fixture, + expectAlpha2: true, + expectedMigrationCount: CURRENT_MIGRATION_COUNT, + }) await traffic(id, fixture) assert.equal(await activeRevision(id), originalRevision) - passed('repeated startup after Alpha 1 upgrade is idempotent') + passed('repeated startup after ' + baseline.name + ' upgrade is idempotent') await command([...nextCompose, 'down', '--remove-orphans'], 180_000) await options.commandWithEnvironment( @@ -282,10 +367,16 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise command, fixture, expectAlpha2: true, + expectedMigrationCount: CURRENT_MIGRATION_COUNT, }) await traffic(restoredId, fixture) assert.equal(await activeRevision(restoredId), originalRevision) - passed('Alpha 1 database and controller backup restores into a fresh Alpha 2 appliance') + passed( + baseline.name + + ' database and controller backup restores into a fresh ' + + baseline.targetName + + ' appliance', + ) await verifyRestoreRollback({ containerId: restoredId, command, @@ -302,6 +393,7 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise command, fixture, expectAlpha2: true, + expectedMigrationCount: CURRENT_MIGRATION_COUNT, }) await traffic(restoredId, fixture) assert.equal(await activeRevision(restoredId), originalRevision) @@ -315,3 +407,12 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise ) } } + +export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise { + return verifyPublishedUpgrade(options, ALPHA1_BASELINE) +} + +export async function verifyAlpha3Upgrade(options: UpgradeSmokeOptions): Promise { + return verifyPublishedUpgrade(options, ALPHA3_BASELINE) +} + diff --git a/scripts/alpha3-upgrade-smoke.ts b/scripts/alpha3-upgrade-smoke.ts new file mode 100644 index 0000000..3666990 --- /dev/null +++ b/scripts/alpha3-upgrade-smoke.ts @@ -0,0 +1,8 @@ +export { + ALPHA3_BASELINE, + ALPHA3_IMAGE, + ALPHA3_REVISION, + verifyAlpha3Upgrade, + type PublishedUpgradeBaseline, + type UpgradeSmokeOptions, +} from './alpha1-upgrade-smoke' From 7ad00362db1bcad48f6f0fb974a89c1907908f5c Mon Sep 17 00:00:00 2001 From: "HomePC\\Kevin" Date: Sun, 13 Sep 2026 00:29:17 +0200 Subject: [PATCH 2/6] test(release): round-trip durable certificate state through production archives --- scripts/appliance-compose-smoke.ts | 26 ++++----- scripts/certificate-smoke.ts | 30 +++++++++++ scripts/certificate-state-restore-smoke.ts | 61 ++++++++++++++++++++++ scripts/controller-state-archive.ts | 24 +++++++++ scripts/production-backup.ts | 25 +-------- 5 files changed, 130 insertions(+), 36 deletions(-) create mode 100644 scripts/certificate-state-restore-smoke.ts create mode 100644 scripts/controller-state-archive.ts diff --git a/scripts/appliance-compose-smoke.ts b/scripts/appliance-compose-smoke.ts index 3c540ef..0abd496 100644 --- a/scripts/appliance-compose-smoke.ts +++ b/scripts/appliance-compose-smoke.ts @@ -10,8 +10,8 @@ import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { restoreSmokeDiagnostic, smokeCompose, smokeDockerArguments } from './smoke-resources' -import { verifyAlpha1Upgrade } from './alpha1-upgrade-smoke' import { buildHttp3Client, requestHttp3Client, assertHttp3Response } from './http3-client' +import { verifyAlpha1Upgrade, verifyAlpha3Upgrade } from './alpha1-upgrade-smoke' const repositoryRoot = fileURLToPath(new URL('..', import.meta.url)) const rootComposeFile = join(repositoryRoot, 'docker-compose.yml') @@ -1194,17 +1194,19 @@ async function runSmoke(): Promise { await restoreLegacyFixture(legacyV1, legacyComposes[1]!, legacyProjects[1]!, 1) await command([...legacyComposes[1]!, 'down', '--volumes', '--remove-orphans'], 180_000) await command([...restoreCompose, 'down', '--volumes', '--remove-orphans'], 180_000) - await verifyAlpha1Upgrade({ - imageTag, - temporaryRoot, - upstreamPort: backendPort, - trafficMarker, - envFile, - environment: scriptEnvironment, - command, - commandWithEnvironment, - passed, - }) + for (const verifyUpgrade of [verifyAlpha1Upgrade, verifyAlpha3Upgrade]) { + await verifyUpgrade({ + imageTag, + temporaryRoot, + upstreamPort: backendPort, + trafficMarker, + envFile, + environment: scriptEnvironment, + command, + commandWithEnvironment, + passed, + }) + } } finally { backend?.stop(true) await commandFails([...compose, 'down', '--volumes', '--remove-orphans'], 180_000) diff --git a/scripts/certificate-smoke.ts b/scripts/certificate-smoke.ts index aeea6cf..b640fe6 100644 --- a/scripts/certificate-smoke.ts +++ b/scripts/certificate-smoke.ts @@ -13,6 +13,7 @@ import { startCertificateDnsFixture } from './certificate-dns-fixture' import { verifyProxyAccessLogs } from './proxy-access-logs-smoke' import { verifyDurableCertificateJob } from './certificate-job-smoke' import { buildHttp3Client, requestHttp3Client, assertHttp3Response } from './http3-client' +import { restoreCertificateStateFixture } from './certificate-state-restore-smoke' import { CERTIFICATE_ERROR_CODES } from '../web/src/config/certificates.config' const repositoryRoot = fileURLToPath(new URL('..', import.meta.url)) @@ -1988,6 +1989,22 @@ async function runSmoke(): Promise { await command(['docker', 'network', 'disconnect', network, pebbleContainer], { timeoutMs: 30_000, }) + + await restoreCertificateStateFixture(command, { + container: runtimeContainer, + volume: stateVolume, + image: runtimeImage, + requiredFiles: [ + 'certificates/certificate-metadata.json', + 'certificates/acme-accounts/staging.json', + 'certificates/' + acmeId + '/candidate.json', + ], + }) + await refreshRuntimeDns() + await waitForRuntimeReady() + passed( + 'production archive restores active material, issued candidate, operation journal and ACME account', + ) await restartRuntime() let restartedCandidate: Record = {} await waitFor(async () => { @@ -2220,6 +2237,19 @@ async function runSmoke(): Promise { await command(['docker', 'network', 'disconnect', network, pebbleContainer], { timeoutMs: 30_000, }) + + await restoreCertificateStateFixture(command, { + container: runtimeContainer, + volume: stateVolume, + image: runtimeImage, + requiredFiles: [ + 'certificates/certificate-metadata.json', + 'certificates/acme-accounts/staging.json', + ], + }) + await refreshRuntimeDns() + await waitForRuntimeReady() + passed('production archive restores encrypted DNS credentials and pending cleanup state') await restartRuntime() await waitFor( async () => (await controllerRequest('/internal/v1/proxy/status')).status === 200, diff --git a/scripts/certificate-state-restore-smoke.ts b/scripts/certificate-state-restore-smoke.ts new file mode 100644 index 0000000..ee87ece --- /dev/null +++ b/scripts/certificate-state-restore-smoke.ts @@ -0,0 +1,61 @@ +import assert from 'node:assert/strict' +import { randomUUID } from 'node:crypto' + +import { stateArchiveExclusions } from './controller-state-archive' + +type Command = (args: string[], options?: { timeoutMs?: number }) => Promise + +export async function restoreCertificateStateFixture( + command: Command, + input: { container: string; volume: string; image: string; requiredFiles: readonly string[] }, +): Promise { + assert.match(input.container, /^rentnerproxy-certificate-smoke-[a-f0-9]{12}-runtime$/u) + assert.match(input.volume, /^rentnerproxy-certificate-smoke-[a-f0-9]{12}-state$/u) + const archiveVolume = input.volume + '-restore-' + randomUUID().replaceAll('-', '') + await command(['docker', 'volume', 'create', archiveVolume]) + try { + await command(['docker', 'stop', input.container], { timeoutMs: 60_000 }) + const run = [ + 'docker', + 'run', + '--rm', + '--entrypoint', + 'sh', + '--volume', + input.volume + ':/state', + '--volume', + archiveVolume + ':/archive', + input.image, + '-ceu', + ] + const exclusions = stateArchiveExclusions + .map((entry) => "--exclude='" + entry + "'") + .join(' ') + const files = await command([ + ...run, + 'tar --create --file=/archive/controller-state.tar --directory=/state ' + + exclusions + + ' .; ' + + 'mkdir /archive/expected; tar --extract --file=/archive/controller-state.tar --directory=/archive/expected; ' + + 'cd /archive/expected; find . -type f -print | LC_ALL=C sort', + ]) + for (const path of input.requiredFiles) { + assert.ok( + files.split('\n').includes('./' + path), + 'Backup omitted required state: ' + path, + ) + } + assert.doesNotMatch(files, /(?:^|\n)\.\/logs\//u) + await command([ + ...run, + 'cd /archive/expected; find . -type f -exec sha256sum {} + | LC_ALL=C sort > /archive/expected.sha256; ' + + 'find /state -mindepth 1 -delete; ' + + 'tar --extract --file=/archive/controller-state.tar --directory=/state; ' + + 'cd /state; find . -type f -exec sha256sum {} + | LC_ALL=C sort > /archive/restored.sha256; ' + + 'cmp /archive/expected.sha256 /archive/restored.sha256', + ]) + await command(['docker', 'start', input.container]) + } finally { + await command(['docker', 'volume', 'rm', archiveVolume]).catch(() => undefined) + } +} diff --git a/scripts/controller-state-archive.ts b/scripts/controller-state-archive.ts new file mode 100644 index 0000000..60346e8 --- /dev/null +++ b/scripts/controller-state-archive.ts @@ -0,0 +1,24 @@ +export const stateArchiveExclusions = [ + './active.conf', + './candidate.conf', + './last-known-good.conf', + './last-good.conf', + './engine.pid', + './host-configs', + './host-configs/**', + './caddy-admin.sock', + './runtime-probe.sock', + './caddy/**/*.sock', + './caddy/**/*.tmp', + './caddy/**/*.lock', + './cache', + './bootstrap', + './runtime', + './tmp', + './run', + './log', + './logs', + '*.log', + '*.log.*', + '.*.tmp', +] diff --git a/scripts/production-backup.ts b/scripts/production-backup.ts index 7a9d640..750dded 100644 --- a/scripts/production-backup.ts +++ b/scripts/production-backup.ts @@ -1,3 +1,4 @@ +import { stateArchiveExclusions } from './controller-state-archive' import { createHash, randomUUID } from 'node:crypto' import { chmod, mkdir, mkdtemp, readFile, rename, rm, stat, writeFile } from 'node:fs/promises' import { isAbsolute, join, resolve } from 'node:path' @@ -12,30 +13,6 @@ const databaseUser = 'rentnerproxy' const statePath = '/var/lib/rentnerproxy/proxy' const stateArchiveName = 'controller-state.tar' const bootstrapScript = '/opt/rentnerproxy/web/docker/web/bootstrap-secrets.mjs' -const stateArchiveExclusions = [ - './active.conf', - './candidate.conf', - './last-known-good.conf', - './last-good.conf', - './engine.pid', - './host-configs', - './host-configs/**', - './caddy-admin.sock', - './runtime-probe.sock', - './caddy/**/*.sock', - './caddy/**/*.tmp', - './caddy/**/*.lock', - './cache', - './bootstrap', - './runtime', - './tmp', - './run', - './log', - './logs', - '*.log', - '*.log.*', - '.*.tmp', -] type CommandOptions = Readonly<{ timeoutMs?: number From 0e51d4bc3b342456d8214f924c96a96634704534 Mon Sep 17 00:00:00 2001 From: "HomePC\\Kevin" Date: Sun, 13 Sep 2026 00:33:08 +0200 Subject: [PATCH 3/6] test: add alpha4 persistence fixture --- scripts/alpha4-persistence-fixture.ts | 537 ++++++++++++++++++++++++++ 1 file changed, 537 insertions(+) create mode 100644 scripts/alpha4-persistence-fixture.ts diff --git a/scripts/alpha4-persistence-fixture.ts b/scripts/alpha4-persistence-fixture.ts new file mode 100644 index 0000000..741025a --- /dev/null +++ b/scripts/alpha4-persistence-fixture.ts @@ -0,0 +1,537 @@ +import { deepStrictEqual } from 'node:assert/strict' +import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto' + +export type Command = (args: string[], timeoutMs?: number) => Promise + +export interface Alpha4CertificateRequest { + readonly name: string + readonly domains: readonly string[] + readonly environment: 'staging' + readonly challengeType: 'http-01' + readonly contactEmail: '' + readonly acceptTerms: true +} + +export interface Alpha4PersistenceFixture { + readonly runId: string + readonly ownerUserId: string + readonly hostId: string + readonly certificateId: string + readonly jobId: string + readonly idempotencyKey: string + readonly operationId: string + readonly eventIds: readonly string[] + readonly domain: string + readonly cursor: string + readonly applicationKeyDigest: string + readonly requestContext: string + readonly request: Alpha4CertificateRequest +} + +export type Alpha4PersistenceSnapshot = Readonly> + +export interface Alpha4PersistenceCommandInput { + readonly command: Command + readonly containerId: string +} + +export interface SeedAlpha4PersistenceFixtureInput extends Alpha4PersistenceCommandInput { + readonly runId: string +} + +export interface AssertAlpha4PersistenceFixtureInput extends Alpha4PersistenceCommandInput { + readonly fixture: Alpha4PersistenceFixture +} + +export interface AssertAlpha4PersistenceSnapshotInput extends AssertAlpha4PersistenceFixtureInput { + readonly expected: Alpha4PersistenceSnapshot +} + +const postgresPasswordFile = '/run/rentnerproxy/postgres/value' +const appEncryptionKeyFile = '/run/rentnerproxy/app-key/value' +const databaseName = 'rentnerproxy' +const databaseUser = 'rentnerproxy' +const futureAttemptAt = '2099-01-01T00:00:00.000Z' +const futureLeaseExpiresAt = '2099-01-02T00:00:00.000Z' +const appEncryptionKeyBytes = 32 +const aesGcmIvBytes = 12 + +function shellQuote(value: string): string { + return "'" + value.replaceAll("'", "'\"'\"'") + "'" +} + +function sqlQuote(value: string): string { + return "'" + value.replaceAll("'", "''") + "'" +} + +function sqlTimestamp(value: Date | string): string { + const serialized = value instanceof Date ? value.toISOString() : value + return `${sqlQuote(serialized)}::timestamptz` +} + +function sqlJson(value: unknown): string { + return `${sqlQuote(JSON.stringify(value))}::jsonb` +} + +function sqlTextArray(values: readonly string[]): string { + return `ARRAY[${values.map(sqlQuote).join(',')}]::text[]` +} + +function uuidV7(): string { + const bytes = randomBytes(16) + const timestamp = BigInt(Date.now()) + bytes[0] = Number((timestamp >> 40n) & 0xffn) + bytes[1] = Number((timestamp >> 32n) & 0xffn) + bytes[2] = Number((timestamp >> 24n) & 0xffn) + bytes[3] = Number((timestamp >> 16n) & 0xffn) + bytes[4] = Number((timestamp >> 8n) & 0xffn) + bytes[5] = Number(timestamp & 0xffn) + bytes[6] = (bytes[6]! & 0x0f) | 0x70 + bytes[8] = (bytes[8]! & 0x3f) | 0x80 + const hex = bytes.toString('hex') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +function validateRunId(runId: string): void { + if (!/^[a-z0-9][a-z0-9_-]{5,63}$/u.test(runId)) { + throw new Error('Alpha 4 fixture runId must be a bounded lowercase identifier.') + } +} + +function validateCommandInput(input: Alpha4PersistenceCommandInput): void { + if (!input.containerId.trim() || input.containerId.length > 256) { + throw new Error('Alpha 4 fixture container id is invalid.') + } +} + +async function inContainer( + command: Command, + containerId: string, + script: string, + timeoutMs = 60_000, +): Promise { + return command(['docker', 'exec', containerId, 'sh', '-ceu', script], timeoutMs) +} + +async function psql( + command: Command, + containerId: string, + statement: string, + timeoutMs = 60_000, +): Promise { + const script = + 'PGPASSWORD="$(cat ' + + postgresPasswordFile + + ')" gosu postgres psql --no-psqlrc --no-password --quiet --set=ON_ERROR_STOP=1 ' + + '--tuples-only --no-align --host=127.0.0.1 --username=' + + databaseUser + + ' --dbname=' + + databaseName + + ' --command=' + + shellQuote(statement) + return inContainer(command, containerId, script, timeoutMs) +} + +function digest(value: unknown): string { + return createHash('sha256') + .update(JSON.stringify(canonicalValue(value))) + .digest('hex') +} + +function canonicalValue(value: unknown): unknown { + if (value instanceof Date) return value.toISOString() + if (Array.isArray(value)) return value.map(canonicalValue) + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, canonicalValue(entry)]), + ) + } + return value +} + +function requireRecord(value: unknown, label: string): Record { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`Alpha 4 fixture returned invalid ${label}.`) + } + return value as Record +} + +function parseJsonRecord(value: string, label: string): Record { + try { + return requireRecord(JSON.parse(value), label) + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Alpha 4 fixture returned invalid ${label}.`, { cause: error }) + } + throw error + } +} + +function readUuidFixtureIds(): { + readonly ownerUserId: string + readonly hostId: string + readonly certificateId: string + readonly jobId: string + readonly idempotencyKey: string + readonly operationId: string + readonly eventIds: readonly [string, string] +} { + return { + ownerUserId: uuidV7(), + hostId: uuidV7(), + certificateId: uuidV7(), + jobId: uuidV7(), + idempotencyKey: uuidV7(), + operationId: uuidV7(), + eventIds: [uuidV7(), uuidV7()], + } +} + +function encryptRequest( + request: Alpha4CertificateRequest, + key: Buffer, + context: string, +): { readonly ciphertext: Buffer; readonly iv: Buffer } { + const iv = randomBytes(aesGcmIvBytes) + const cipher = createCipheriv('aes-256-gcm', key, iv) + cipher.setAAD(Buffer.from(context, 'utf8')) + const ciphertext = Buffer.concat([ + cipher.update(JSON.stringify(request), 'utf8'), + cipher.final(), + cipher.getAuthTag(), + ]) + return { ciphertext, iv } +} + +async function readApplicationKey( + command: Command, + containerId: string, +): Promise<{ readonly bytes: Buffer; readonly digest: string }> { + const encoded = (await inContainer(command, containerId, `cat ${appEncryptionKeyFile}`)).trim() + const bytes = Buffer.from(encoded, 'base64') + if (bytes.byteLength !== appEncryptionKeyBytes || bytes.toString('base64') !== encoded) { + throw new Error('Alpha 4 fixture application encryption key is invalid.') + } + return { bytes, digest: createHash('sha256').update(bytes).digest('hex') } +} + +function isCursor(value: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}:\d+$/iu.test(value) +} + +async function readOrCreateCursor( + command: Command, + containerId: string, + now: Date, +): Promise<{ readonly cursor: string; readonly statement: string }> { + const existing = ( + await psql( + command, + containerId, + "select coalesce(cursor, '') from rentnerproxy.certificate_event_cursor where id = 1", + ) + ).trim() + if (isCursor(existing)) { + return { + cursor: existing, + statement: + 'insert into rentnerproxy.certificate_event_cursor (id, cursor, updated_at) values (1, ' + + sqlQuote(existing) + + ', ' + + sqlTimestamp(now) + + ') on conflict (id) do nothing', + } + } + const cursor = `${uuidV7()}:0` + return { + cursor, + statement: + 'insert into rentnerproxy.certificate_event_cursor (id, cursor, updated_at) values (1, ' + + sqlQuote(cursor) + + ', ' + + sqlTimestamp(now) + + ') on conflict (id) do update set cursor = excluded.cursor, updated_at = excluded.updated_at', + } +} + +export async function seedAlpha4PersistenceFixture( + input: SeedAlpha4PersistenceFixtureInput, +): Promise { + validateCommandInput(input) + validateRunId(input.runId) + const key = await readApplicationKey(input.command, input.containerId) + const ids = readUuidFixtureIds() + const now = new Date() + const operationStartedAt = new Date(now.getTime() - 10 * 60_000) + const lastSuccessAt = new Date(now.getTime() - 24 * 60 * 60_000) + const lastAttemptAt = new Date(now.getTime() - 5 * 60_000) + const certificateCreatedAt = new Date(now.getTime() - 20 * 60_000) + const jobCreatedAt = new Date(now.getTime() - 15 * 60_000) + const issuedAt = new Date(now.getTime() - 30 * 24 * 60 * 60_000) + const expiresAt = new Date(now.getTime() + 60 * 24 * 60 * 60_000) + const nextRenewalAt = new Date(now.getTime() + 30 * 24 * 60 * 60_000) + const identityDigest = digest({ runId: input.runId, ownerUserId: ids.ownerUserId }) + const domain = `a4-${identityDigest.slice(0, 32)}.example.com` + const request: Alpha4CertificateRequest = { + name: `Alpha 4 persistence ${input.runId}`, + domains: [domain], + environment: 'staging', + challengeType: 'http-01', + contactEmail: '', + acceptTerms: true, + } + const requestContext = `certificate-binding-job:${ids.jobId}` + const encrypted = encryptRequest(request, key.bytes, requestContext) + const certificateFingerprint = `sha256:${digest({ runId: input.runId, certificate: true })}` + const host = { + id: ids.hostId, + forwardScheme: 'http', + forwardHost: '127.0.0.1', + forwardPort: 8080, + enabled: false, + certificateId: ids.certificateId, + forceHttps: false, + verifyUpstreamTls: true, + upstreamTlsServerName: null, + trustedCaId: null, + accessPolicyId: null, + createdAt: now, + updatedAt: now, + } + const hostRevision = digest({ host, domains: [domain], httpSettings: {} }) + const requestDigest = digest({ proxyHostId: ids.hostId, host: null, request }) + const cursor = await readOrCreateCursor(input.command, input.containerId, now) + const operation = { + id: ids.operationId, + kind: 'renew', + stage: 'retry_scheduled', + startedAt: operationStartedAt.toISOString(), + updatedAt: now.toISOString(), + } + const statements = [ + 'begin', + 'insert into rentnerproxy.users (id, display_name, email, status, created_at, updated_at) values (' + + sqlQuote(ids.ownerUserId) + + ', ' + + sqlQuote(`Alpha 4 Fixture Owner ${input.runId}`) + + ', ' + + sqlQuote(`alpha4-${identityDigest.slice(0, 24)}@fixture.invalid`) + + ", 'active', " + + sqlTimestamp(certificateCreatedAt) + + ', ' + + sqlTimestamp(now) + + ')', + 'insert into rentnerproxy.user_roles (user_id, role_id) select ' + + sqlQuote(ids.ownerUserId) + + ", id from rentnerproxy.roles where key = 'owner'", + 'insert into rentnerproxy.certificates (id, name, source, environment, status, operation, current_operation, challenge_type, issued_at, expires_at, issuer, fingerprint, candidate, dns_cleanup_pending, last_error_code, last_activated_at, last_error_at, next_attempt_at, attempt_count, last_attempt_at, last_success_at, next_renewal_at, controller_updated_at, created_at, updated_at) values (' + + sqlQuote(ids.certificateId) + + ', ' + + sqlQuote(`Alpha 4 persistence ${input.runId}`) + + ", 'acme', 'staging', 'valid', 'renewing', " + + sqlJson(operation) + + ", 'http-01', " + + sqlTimestamp(issuedAt) + + ', ' + + sqlTimestamp(expiresAt) + + ', ' + + sqlQuote('Alpha 4 Fixture CA') + + ', ' + + sqlQuote(certificateFingerprint) + + ', null, false, ' + + sqlQuote('acme_failed') + + ', ' + + sqlTimestamp(lastSuccessAt) + + ', ' + + sqlTimestamp(lastAttemptAt) + + ', ' + + sqlTimestamp(futureAttemptAt) + + ', 2, ' + + sqlTimestamp(lastAttemptAt) + + ', ' + + sqlTimestamp(lastSuccessAt) + + ', ' + + sqlTimestamp(nextRenewalAt) + + ', ' + + sqlTimestamp(now) + + ', ' + + sqlTimestamp(certificateCreatedAt) + + ', ' + + sqlTimestamp(now) + + ')', + 'insert into rentnerproxy.certificate_domains (certificate_id, domain, created_at) values (' + + sqlQuote(ids.certificateId) + + ', ' + + sqlQuote(domain) + + ', ' + + sqlTimestamp(certificateCreatedAt) + + ')', + 'insert into rentnerproxy.proxy_hosts (id, forward_scheme, forward_host, forward_port, enabled, certificate_id, force_https, verify_upstream_tls, upstream_tls_server_name, trusted_ca_id, access_policy_id, created_at, updated_at) values (' + + sqlQuote(ids.hostId) + + ", 'http', '127.0.0.1', 8080, false, " + + sqlQuote(ids.certificateId) + + ', false, true, null, null, null, ' + + sqlTimestamp(now) + + ', ' + + sqlTimestamp(now) + + ')', + 'insert into rentnerproxy.host_domains (proxy_host_id, domain, created_at) values (' + + sqlQuote(ids.hostId) + + ', ' + + sqlQuote(domain) + + ', ' + + sqlTimestamp(now) + + ')', + 'insert into rentnerproxy.certificate_binding_jobs (id, actor_user_id, proxy_host_id, certificate_id, idempotency_key, request_digest, domains, required_permissions, host_revision, assigned_revision, desired_enabled, desired_force_https, request_ciphertext, request_iv, stage, controller_stage, controller_operation_id, last_error_code, attempt_count, retry_requested, next_attempt_at, lease_token, lease_expires_at, created_at, updated_at) values (' + + sqlQuote(ids.jobId) + + ', ' + + sqlQuote(ids.ownerUserId) + + ', ' + + sqlQuote(ids.hostId) + + ', ' + + sqlQuote(ids.certificateId) + + ', ' + + sqlQuote(ids.idempotencyKey) + + ', ' + + sqlQuote(requestDigest) + + ', ' + + sqlTextArray([domain]) + + ', ' + + sqlTextArray(['proxy_hosts.update', 'certificates.issue']) + + ', ' + + sqlQuote(hostRevision) + + ', ' + + sqlQuote(hostRevision) + + ', false, false, decode(' + + sqlQuote(encrypted.ciphertext.toString('base64')) + + ", 'base64'), decode(" + + sqlQuote(encrypted.iv.toString('base64')) + + ", 'base64'), 'issuing', 'retry_scheduled', " + + sqlQuote(ids.operationId) + + ", 'acme_failed', 2, true, " + + sqlTimestamp(futureAttemptAt) + + ', ' + + sqlQuote(uuidV7()) + + ', ' + + sqlTimestamp(futureLeaseExpiresAt) + + ', ' + + sqlTimestamp(jobCreatedAt) + + ', ' + + sqlTimestamp(now) + + ')', + 'insert into rentnerproxy.certificate_event_receipts (event_id, operation_id, certificate_id, kind, stage, occurred_at, error_code, received_at) values (' + + sqlQuote(ids.eventIds[0]) + + ', ' + + sqlQuote(ids.operationId) + + ', ' + + sqlQuote(ids.certificateId) + + ", 'started', 'creating_order', " + + sqlTimestamp(operationStartedAt) + + ', null, ' + + sqlTimestamp(now) + + '), (' + + sqlQuote(ids.eventIds[1]) + + ', ' + + sqlQuote(ids.operationId) + + ', ' + + sqlQuote(ids.certificateId) + + ", 'retry_scheduled', 'retry_scheduled', " + + sqlTimestamp(lastAttemptAt) + + ', ' + + sqlQuote('acme_failed') + + ', ' + + sqlTimestamp(now) + + ')', + cursor.statement, + 'commit', + ] + await psql(input.command, input.containerId, statements.join(';\n') + ';', 60_000) + return { + runId: input.runId, + ownerUserId: ids.ownerUserId, + hostId: ids.hostId, + certificateId: ids.certificateId, + jobId: ids.jobId, + idempotencyKey: ids.idempotencyKey, + operationId: ids.operationId, + eventIds: ids.eventIds, + domain, + cursor: cursor.cursor, + applicationKeyDigest: key.digest, + requestContext, + request, + } +} + +function snapshotStatement(fixture: Alpha4PersistenceFixture): string { + return `select jsonb_build_object( + 'fixtureVersion', 1, + 'user', (select to_jsonb(u) - 'password_hash' from rentnerproxy.users u where u.id = ${sqlQuote(fixture.ownerUserId)}), + 'userRoles', (select coalesce(jsonb_agg(to_jsonb(ur) order by ur.role_id), '[]'::jsonb) from rentnerproxy.user_roles ur where ur.user_id = ${sqlQuote(fixture.ownerUserId)}), + 'certificate', (select to_jsonb(c) from rentnerproxy.certificates c where c.id = ${sqlQuote(fixture.certificateId)}), + 'certificateDomains', (select coalesce(jsonb_agg(to_jsonb(d) order by d.domain, d.id), '[]'::jsonb) from rentnerproxy.certificate_domains d where d.certificate_id = ${sqlQuote(fixture.certificateId)}), + 'proxyHost', (select to_jsonb(h) from rentnerproxy.proxy_hosts h where h.id = ${sqlQuote(fixture.hostId)}), + 'hostDomains', (select coalesce(jsonb_agg(to_jsonb(d) order by d.domain, d.id), '[]'::jsonb) from rentnerproxy.host_domains d where d.proxy_host_id = ${sqlQuote(fixture.hostId)}), + 'job', (select (to_jsonb(j) - 'request_ciphertext' - 'request_iv') || jsonb_build_object('request_ciphertext_bytes', octet_length(j.request_ciphertext), 'request_ciphertext_md5', md5(encode(j.request_ciphertext, 'hex')), 'request_iv_bytes', octet_length(j.request_iv), 'request_iv_md5', md5(encode(j.request_iv, 'hex'))) from rentnerproxy.certificate_binding_jobs j where j.id = ${sqlQuote(fixture.jobId)}), + 'eventReceipts', (select coalesce(jsonb_agg(to_jsonb(r) order by r.event_id), '[]'::jsonb) from rentnerproxy.certificate_event_receipts r where r.event_id in (${fixture.eventIds.map(sqlQuote).join(',')})), + 'eventCursor', (select to_jsonb(c) from rentnerproxy.certificate_event_cursor c where c.id = 1) + )::text` +} + +export async function readAlpha4PersistenceSnapshot( + input: AssertAlpha4PersistenceFixtureInput, +): Promise { + validateCommandInput(input) + const output = await psql(input.command, input.containerId, snapshotStatement(input.fixture)) + return parseJsonRecord(output, 'persistence snapshot') +} + +export async function assertAlpha4PersistenceFixture( + input: AssertAlpha4PersistenceSnapshotInput, +): Promise { + const actual = await readAlpha4PersistenceSnapshot(input) + try { + deepStrictEqual(actual, input.expected) + } catch { + throw new Error('Alpha 4 persistence fixture snapshot did not survive backup restore.') + } +} + +export async function assertAlpha4PersistenceRequestDecrypts( + input: AssertAlpha4PersistenceFixtureInput, +): Promise { + validateCommandInput(input) + try { + const key = await readApplicationKey(input.command, input.containerId) + if (key.digest !== input.fixture.applicationKeyDigest) { + throw new Error('application key changed') + } + const encoded = ( + await psql( + input.command, + input.containerId, + `select encode(request_ciphertext, 'base64') || E'\\t' || encode(request_iv, 'base64') from rentnerproxy.certificate_binding_jobs where id = ${sqlQuote(input.fixture.jobId)}`, + ) + ).trim() + const [ciphertextEncoded, ivEncoded] = encoded.split('\t') + if (!ciphertextEncoded || !ivEncoded) throw new Error('encrypted request missing') + const ciphertext = Buffer.from(ciphertextEncoded, 'base64') + const iv = Buffer.from(ivEncoded, 'base64') + if (ciphertext.byteLength < 17 || iv.byteLength !== aesGcmIvBytes) { + throw new Error('encrypted request invalid') + } + const decipher = createDecipheriv('aes-256-gcm', key.bytes, iv) + decipher.setAAD(Buffer.from(input.fixture.requestContext, 'utf8')) + decipher.setAuthTag(ciphertext.subarray(-16)) + const plaintext = Buffer.concat([ + decipher.update(ciphertext.subarray(0, -16)), + decipher.final(), + ]).toString('utf8') + const request = JSON.parse(plaintext) + deepStrictEqual(request, input.fixture.request) + } catch { + throw new Error('Alpha 4 persistence fixture encrypted request could not be decrypted.') + } +} From d75f1a0ef129d0831c2f8abfa8b34609a384d290 Mon Sep 17 00:00:00 2001 From: "HomePC\\Kevin" Date: Sun, 13 Sep 2026 00:43:22 +0200 Subject: [PATCH 4/6] refactor: split alpha4 persistence fixture --- scripts/alpha4-persistence-fixture.ts | 537 ++----------------------- scripts/alpha4-persistence/crypto.ts | 67 +++ scripts/alpha4-persistence/seed-sql.ts | 193 +++++++++ scripts/alpha4-persistence/seed.ts | 107 +++++ scripts/alpha4-persistence/storage.ts | 163 ++++++++ scripts/alpha4-persistence/types.ts | 55 +++ 6 files changed, 629 insertions(+), 493 deletions(-) create mode 100644 scripts/alpha4-persistence/crypto.ts create mode 100644 scripts/alpha4-persistence/seed-sql.ts create mode 100644 scripts/alpha4-persistence/seed.ts create mode 100644 scripts/alpha4-persistence/storage.ts create mode 100644 scripts/alpha4-persistence/types.ts diff --git a/scripts/alpha4-persistence-fixture.ts b/scripts/alpha4-persistence-fixture.ts index 741025a..4526fb0 100644 --- a/scripts/alpha4-persistence-fixture.ts +++ b/scripts/alpha4-persistence-fixture.ts @@ -1,102 +1,24 @@ import { deepStrictEqual } from 'node:assert/strict' -import { createCipheriv, createDecipheriv, createHash, randomBytes } from 'node:crypto' +import { createDecipheriv } from 'node:crypto' + +import { decodeApplicationKey } from './alpha4-persistence/crypto' +import { seedAlpha4PersistenceFixture } from './alpha4-persistence/seed' +import { + readAlpha4PersistenceSnapshot as readSnapshotFromStorage, + readContainerFile, + readEncryptedRequestParts, +} from './alpha4-persistence/storage' +import type { + Alpha4PersistenceCommandInput, + Alpha4PersistenceFixture, + Alpha4PersistenceSnapshot, + AssertAlpha4PersistenceFixtureInput, + AssertAlpha4PersistenceSnapshotInput, + Command, + SeedAlpha4PersistenceFixtureInput, +} from './alpha4-persistence/types' -export type Command = (args: string[], timeoutMs?: number) => Promise - -export interface Alpha4CertificateRequest { - readonly name: string - readonly domains: readonly string[] - readonly environment: 'staging' - readonly challengeType: 'http-01' - readonly contactEmail: '' - readonly acceptTerms: true -} - -export interface Alpha4PersistenceFixture { - readonly runId: string - readonly ownerUserId: string - readonly hostId: string - readonly certificateId: string - readonly jobId: string - readonly idempotencyKey: string - readonly operationId: string - readonly eventIds: readonly string[] - readonly domain: string - readonly cursor: string - readonly applicationKeyDigest: string - readonly requestContext: string - readonly request: Alpha4CertificateRequest -} - -export type Alpha4PersistenceSnapshot = Readonly> - -export interface Alpha4PersistenceCommandInput { - readonly command: Command - readonly containerId: string -} - -export interface SeedAlpha4PersistenceFixtureInput extends Alpha4PersistenceCommandInput { - readonly runId: string -} - -export interface AssertAlpha4PersistenceFixtureInput extends Alpha4PersistenceCommandInput { - readonly fixture: Alpha4PersistenceFixture -} - -export interface AssertAlpha4PersistenceSnapshotInput extends AssertAlpha4PersistenceFixtureInput { - readonly expected: Alpha4PersistenceSnapshot -} - -const postgresPasswordFile = '/run/rentnerproxy/postgres/value' const appEncryptionKeyFile = '/run/rentnerproxy/app-key/value' -const databaseName = 'rentnerproxy' -const databaseUser = 'rentnerproxy' -const futureAttemptAt = '2099-01-01T00:00:00.000Z' -const futureLeaseExpiresAt = '2099-01-02T00:00:00.000Z' -const appEncryptionKeyBytes = 32 -const aesGcmIvBytes = 12 - -function shellQuote(value: string): string { - return "'" + value.replaceAll("'", "'\"'\"'") + "'" -} - -function sqlQuote(value: string): string { - return "'" + value.replaceAll("'", "''") + "'" -} - -function sqlTimestamp(value: Date | string): string { - const serialized = value instanceof Date ? value.toISOString() : value - return `${sqlQuote(serialized)}::timestamptz` -} - -function sqlJson(value: unknown): string { - return `${sqlQuote(JSON.stringify(value))}::jsonb` -} - -function sqlTextArray(values: readonly string[]): string { - return `ARRAY[${values.map(sqlQuote).join(',')}]::text[]` -} - -function uuidV7(): string { - const bytes = randomBytes(16) - const timestamp = BigInt(Date.now()) - bytes[0] = Number((timestamp >> 40n) & 0xffn) - bytes[1] = Number((timestamp >> 32n) & 0xffn) - bytes[2] = Number((timestamp >> 24n) & 0xffn) - bytes[3] = Number((timestamp >> 16n) & 0xffn) - bytes[4] = Number((timestamp >> 8n) & 0xffn) - bytes[5] = Number(timestamp & 0xffn) - bytes[6] = (bytes[6]! & 0x0f) | 0x70 - bytes[8] = (bytes[8]! & 0x3f) | 0x80 - const hex = bytes.toString('hex') - return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` -} - -function validateRunId(runId: string): void { - if (!/^[a-z0-9][a-z0-9_-]{5,63}$/u.test(runId)) { - throw new Error('Alpha 4 fixture runId must be a bounded lowercase identifier.') - } -} function validateCommandInput(input: Alpha4PersistenceCommandInput): void { if (!input.containerId.trim() || input.containerId.length > 256) { @@ -104,393 +26,17 @@ function validateCommandInput(input: Alpha4PersistenceCommandInput): void { } } -async function inContainer( - command: Command, - containerId: string, - script: string, - timeoutMs = 60_000, -): Promise { - return command(['docker', 'exec', containerId, 'sh', '-ceu', script], timeoutMs) -} - -async function psql( - command: Command, - containerId: string, - statement: string, - timeoutMs = 60_000, -): Promise { - const script = - 'PGPASSWORD="$(cat ' + - postgresPasswordFile + - ')" gosu postgres psql --no-psqlrc --no-password --quiet --set=ON_ERROR_STOP=1 ' + - '--tuples-only --no-align --host=127.0.0.1 --username=' + - databaseUser + - ' --dbname=' + - databaseName + - ' --command=' + - shellQuote(statement) - return inContainer(command, containerId, script, timeoutMs) -} - -function digest(value: unknown): string { - return createHash('sha256') - .update(JSON.stringify(canonicalValue(value))) - .digest('hex') -} - -function canonicalValue(value: unknown): unknown { - if (value instanceof Date) return value.toISOString() - if (Array.isArray(value)) return value.map(canonicalValue) - if (value !== null && typeof value === 'object') { - return Object.fromEntries( - Object.entries(value as Record) - .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) - .map(([key, entry]) => [key, canonicalValue(entry)]), - ) - } - return value -} - -function requireRecord(value: unknown, label: string): Record { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`Alpha 4 fixture returned invalid ${label}.`) - } - return value as Record -} - -function parseJsonRecord(value: string, label: string): Record { - try { - return requireRecord(JSON.parse(value), label) - } catch (error) { - if (error instanceof SyntaxError) { - throw new Error(`Alpha 4 fixture returned invalid ${label}.`, { cause: error }) - } - throw error - } -} - -function readUuidFixtureIds(): { - readonly ownerUserId: string - readonly hostId: string - readonly certificateId: string - readonly jobId: string - readonly idempotencyKey: string - readonly operationId: string - readonly eventIds: readonly [string, string] -} { - return { - ownerUserId: uuidV7(), - hostId: uuidV7(), - certificateId: uuidV7(), - jobId: uuidV7(), - idempotencyKey: uuidV7(), - operationId: uuidV7(), - eventIds: [uuidV7(), uuidV7()], - } -} - -function encryptRequest( - request: Alpha4CertificateRequest, - key: Buffer, - context: string, -): { readonly ciphertext: Buffer; readonly iv: Buffer } { - const iv = randomBytes(aesGcmIvBytes) - const cipher = createCipheriv('aes-256-gcm', key, iv) - cipher.setAAD(Buffer.from(context, 'utf8')) - const ciphertext = Buffer.concat([ - cipher.update(JSON.stringify(request), 'utf8'), - cipher.final(), - cipher.getAuthTag(), - ]) - return { ciphertext, iv } -} - -async function readApplicationKey( - command: Command, - containerId: string, -): Promise<{ readonly bytes: Buffer; readonly digest: string }> { - const encoded = (await inContainer(command, containerId, `cat ${appEncryptionKeyFile}`)).trim() - const bytes = Buffer.from(encoded, 'base64') - if (bytes.byteLength !== appEncryptionKeyBytes || bytes.toString('base64') !== encoded) { - throw new Error('Alpha 4 fixture application encryption key is invalid.') - } - return { bytes, digest: createHash('sha256').update(bytes).digest('hex') } -} - -function isCursor(value: string): boolean { - return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}:\d+$/iu.test(value) -} - -async function readOrCreateCursor( - command: Command, - containerId: string, - now: Date, -): Promise<{ readonly cursor: string; readonly statement: string }> { - const existing = ( - await psql( - command, - containerId, - "select coalesce(cursor, '') from rentnerproxy.certificate_event_cursor where id = 1", - ) - ).trim() - if (isCursor(existing)) { - return { - cursor: existing, - statement: - 'insert into rentnerproxy.certificate_event_cursor (id, cursor, updated_at) values (1, ' + - sqlQuote(existing) + - ', ' + - sqlTimestamp(now) + - ') on conflict (id) do nothing', - } - } - const cursor = `${uuidV7()}:0` - return { - cursor, - statement: - 'insert into rentnerproxy.certificate_event_cursor (id, cursor, updated_at) values (1, ' + - sqlQuote(cursor) + - ', ' + - sqlTimestamp(now) + - ') on conflict (id) do update set cursor = excluded.cursor, updated_at = excluded.updated_at', - } -} - -export async function seedAlpha4PersistenceFixture( - input: SeedAlpha4PersistenceFixtureInput, -): Promise { - validateCommandInput(input) - validateRunId(input.runId) - const key = await readApplicationKey(input.command, input.containerId) - const ids = readUuidFixtureIds() - const now = new Date() - const operationStartedAt = new Date(now.getTime() - 10 * 60_000) - const lastSuccessAt = new Date(now.getTime() - 24 * 60 * 60_000) - const lastAttemptAt = new Date(now.getTime() - 5 * 60_000) - const certificateCreatedAt = new Date(now.getTime() - 20 * 60_000) - const jobCreatedAt = new Date(now.getTime() - 15 * 60_000) - const issuedAt = new Date(now.getTime() - 30 * 24 * 60 * 60_000) - const expiresAt = new Date(now.getTime() + 60 * 24 * 60 * 60_000) - const nextRenewalAt = new Date(now.getTime() + 30 * 24 * 60 * 60_000) - const identityDigest = digest({ runId: input.runId, ownerUserId: ids.ownerUserId }) - const domain = `a4-${identityDigest.slice(0, 32)}.example.com` - const request: Alpha4CertificateRequest = { - name: `Alpha 4 persistence ${input.runId}`, - domains: [domain], - environment: 'staging', - challengeType: 'http-01', - contactEmail: '', - acceptTerms: true, - } - const requestContext = `certificate-binding-job:${ids.jobId}` - const encrypted = encryptRequest(request, key.bytes, requestContext) - const certificateFingerprint = `sha256:${digest({ runId: input.runId, certificate: true })}` - const host = { - id: ids.hostId, - forwardScheme: 'http', - forwardHost: '127.0.0.1', - forwardPort: 8080, - enabled: false, - certificateId: ids.certificateId, - forceHttps: false, - verifyUpstreamTls: true, - upstreamTlsServerName: null, - trustedCaId: null, - accessPolicyId: null, - createdAt: now, - updatedAt: now, - } - const hostRevision = digest({ host, domains: [domain], httpSettings: {} }) - const requestDigest = digest({ proxyHostId: ids.hostId, host: null, request }) - const cursor = await readOrCreateCursor(input.command, input.containerId, now) - const operation = { - id: ids.operationId, - kind: 'renew', - stage: 'retry_scheduled', - startedAt: operationStartedAt.toISOString(), - updatedAt: now.toISOString(), - } - const statements = [ - 'begin', - 'insert into rentnerproxy.users (id, display_name, email, status, created_at, updated_at) values (' + - sqlQuote(ids.ownerUserId) + - ', ' + - sqlQuote(`Alpha 4 Fixture Owner ${input.runId}`) + - ', ' + - sqlQuote(`alpha4-${identityDigest.slice(0, 24)}@fixture.invalid`) + - ", 'active', " + - sqlTimestamp(certificateCreatedAt) + - ', ' + - sqlTimestamp(now) + - ')', - 'insert into rentnerproxy.user_roles (user_id, role_id) select ' + - sqlQuote(ids.ownerUserId) + - ", id from rentnerproxy.roles where key = 'owner'", - 'insert into rentnerproxy.certificates (id, name, source, environment, status, operation, current_operation, challenge_type, issued_at, expires_at, issuer, fingerprint, candidate, dns_cleanup_pending, last_error_code, last_activated_at, last_error_at, next_attempt_at, attempt_count, last_attempt_at, last_success_at, next_renewal_at, controller_updated_at, created_at, updated_at) values (' + - sqlQuote(ids.certificateId) + - ', ' + - sqlQuote(`Alpha 4 persistence ${input.runId}`) + - ", 'acme', 'staging', 'valid', 'renewing', " + - sqlJson(operation) + - ", 'http-01', " + - sqlTimestamp(issuedAt) + - ', ' + - sqlTimestamp(expiresAt) + - ', ' + - sqlQuote('Alpha 4 Fixture CA') + - ', ' + - sqlQuote(certificateFingerprint) + - ', null, false, ' + - sqlQuote('acme_failed') + - ', ' + - sqlTimestamp(lastSuccessAt) + - ', ' + - sqlTimestamp(lastAttemptAt) + - ', ' + - sqlTimestamp(futureAttemptAt) + - ', 2, ' + - sqlTimestamp(lastAttemptAt) + - ', ' + - sqlTimestamp(lastSuccessAt) + - ', ' + - sqlTimestamp(nextRenewalAt) + - ', ' + - sqlTimestamp(now) + - ', ' + - sqlTimestamp(certificateCreatedAt) + - ', ' + - sqlTimestamp(now) + - ')', - 'insert into rentnerproxy.certificate_domains (certificate_id, domain, created_at) values (' + - sqlQuote(ids.certificateId) + - ', ' + - sqlQuote(domain) + - ', ' + - sqlTimestamp(certificateCreatedAt) + - ')', - 'insert into rentnerproxy.proxy_hosts (id, forward_scheme, forward_host, forward_port, enabled, certificate_id, force_https, verify_upstream_tls, upstream_tls_server_name, trusted_ca_id, access_policy_id, created_at, updated_at) values (' + - sqlQuote(ids.hostId) + - ", 'http', '127.0.0.1', 8080, false, " + - sqlQuote(ids.certificateId) + - ', false, true, null, null, null, ' + - sqlTimestamp(now) + - ', ' + - sqlTimestamp(now) + - ')', - 'insert into rentnerproxy.host_domains (proxy_host_id, domain, created_at) values (' + - sqlQuote(ids.hostId) + - ', ' + - sqlQuote(domain) + - ', ' + - sqlTimestamp(now) + - ')', - 'insert into rentnerproxy.certificate_binding_jobs (id, actor_user_id, proxy_host_id, certificate_id, idempotency_key, request_digest, domains, required_permissions, host_revision, assigned_revision, desired_enabled, desired_force_https, request_ciphertext, request_iv, stage, controller_stage, controller_operation_id, last_error_code, attempt_count, retry_requested, next_attempt_at, lease_token, lease_expires_at, created_at, updated_at) values (' + - sqlQuote(ids.jobId) + - ', ' + - sqlQuote(ids.ownerUserId) + - ', ' + - sqlQuote(ids.hostId) + - ', ' + - sqlQuote(ids.certificateId) + - ', ' + - sqlQuote(ids.idempotencyKey) + - ', ' + - sqlQuote(requestDigest) + - ', ' + - sqlTextArray([domain]) + - ', ' + - sqlTextArray(['proxy_hosts.update', 'certificates.issue']) + - ', ' + - sqlQuote(hostRevision) + - ', ' + - sqlQuote(hostRevision) + - ', false, false, decode(' + - sqlQuote(encrypted.ciphertext.toString('base64')) + - ", 'base64'), decode(" + - sqlQuote(encrypted.iv.toString('base64')) + - ", 'base64'), 'issuing', 'retry_scheduled', " + - sqlQuote(ids.operationId) + - ", 'acme_failed', 2, true, " + - sqlTimestamp(futureAttemptAt) + - ', ' + - sqlQuote(uuidV7()) + - ', ' + - sqlTimestamp(futureLeaseExpiresAt) + - ', ' + - sqlTimestamp(jobCreatedAt) + - ', ' + - sqlTimestamp(now) + - ')', - 'insert into rentnerproxy.certificate_event_receipts (event_id, operation_id, certificate_id, kind, stage, occurred_at, error_code, received_at) values (' + - sqlQuote(ids.eventIds[0]) + - ', ' + - sqlQuote(ids.operationId) + - ', ' + - sqlQuote(ids.certificateId) + - ", 'started', 'creating_order', " + - sqlTimestamp(operationStartedAt) + - ', null, ' + - sqlTimestamp(now) + - '), (' + - sqlQuote(ids.eventIds[1]) + - ', ' + - sqlQuote(ids.operationId) + - ', ' + - sqlQuote(ids.certificateId) + - ", 'retry_scheduled', 'retry_scheduled', " + - sqlTimestamp(lastAttemptAt) + - ', ' + - sqlQuote('acme_failed') + - ', ' + - sqlTimestamp(now) + - ')', - cursor.statement, - 'commit', - ] - await psql(input.command, input.containerId, statements.join(';\n') + ';', 60_000) - return { - runId: input.runId, - ownerUserId: ids.ownerUserId, - hostId: ids.hostId, - certificateId: ids.certificateId, - jobId: ids.jobId, - idempotencyKey: ids.idempotencyKey, - operationId: ids.operationId, - eventIds: ids.eventIds, - domain, - cursor: cursor.cursor, - applicationKeyDigest: key.digest, - requestContext, - request, - } -} - -function snapshotStatement(fixture: Alpha4PersistenceFixture): string { - return `select jsonb_build_object( - 'fixtureVersion', 1, - 'user', (select to_jsonb(u) - 'password_hash' from rentnerproxy.users u where u.id = ${sqlQuote(fixture.ownerUserId)}), - 'userRoles', (select coalesce(jsonb_agg(to_jsonb(ur) order by ur.role_id), '[]'::jsonb) from rentnerproxy.user_roles ur where ur.user_id = ${sqlQuote(fixture.ownerUserId)}), - 'certificate', (select to_jsonb(c) from rentnerproxy.certificates c where c.id = ${sqlQuote(fixture.certificateId)}), - 'certificateDomains', (select coalesce(jsonb_agg(to_jsonb(d) order by d.domain, d.id), '[]'::jsonb) from rentnerproxy.certificate_domains d where d.certificate_id = ${sqlQuote(fixture.certificateId)}), - 'proxyHost', (select to_jsonb(h) from rentnerproxy.proxy_hosts h where h.id = ${sqlQuote(fixture.hostId)}), - 'hostDomains', (select coalesce(jsonb_agg(to_jsonb(d) order by d.domain, d.id), '[]'::jsonb) from rentnerproxy.host_domains d where d.proxy_host_id = ${sqlQuote(fixture.hostId)}), - 'job', (select (to_jsonb(j) - 'request_ciphertext' - 'request_iv') || jsonb_build_object('request_ciphertext_bytes', octet_length(j.request_ciphertext), 'request_ciphertext_md5', md5(encode(j.request_ciphertext, 'hex')), 'request_iv_bytes', octet_length(j.request_iv), 'request_iv_md5', md5(encode(j.request_iv, 'hex'))) from rentnerproxy.certificate_binding_jobs j where j.id = ${sqlQuote(fixture.jobId)}), - 'eventReceipts', (select coalesce(jsonb_agg(to_jsonb(r) order by r.event_id), '[]'::jsonb) from rentnerproxy.certificate_event_receipts r where r.event_id in (${fixture.eventIds.map(sqlQuote).join(',')})), - 'eventCursor', (select to_jsonb(c) from rentnerproxy.certificate_event_cursor c where c.id = 1) - )::text` -} - export async function readAlpha4PersistenceSnapshot( input: AssertAlpha4PersistenceFixtureInput, ): Promise { validateCommandInput(input) - const output = await psql(input.command, input.containerId, snapshotStatement(input.fixture)) - return parseJsonRecord(output, 'persistence snapshot') + return readSnapshotFromStorage(input.command, input.containerId, input.fixture) } export async function assertAlpha4PersistenceFixture( input: AssertAlpha4PersistenceSnapshotInput, ): Promise { + validateCommandInput(input) const actual = await readAlpha4PersistenceSnapshot(input) try { deepStrictEqual(actual, input.expected) @@ -504,34 +50,39 @@ export async function assertAlpha4PersistenceRequestDecrypts( ): Promise { validateCommandInput(input) try { - const key = await readApplicationKey(input.command, input.containerId) + const encodedKey = ( + await readContainerFile(input.command, input.containerId, appEncryptionKeyFile) + ).trim() + const key = decodeApplicationKey(encodedKey) if (key.digest !== input.fixture.applicationKeyDigest) { throw new Error('application key changed') } - const encoded = ( - await psql( - input.command, - input.containerId, - `select encode(request_ciphertext, 'base64') || E'\\t' || encode(request_iv, 'base64') from rentnerproxy.certificate_binding_jobs where id = ${sqlQuote(input.fixture.jobId)}`, - ) - ).trim() - const [ciphertextEncoded, ivEncoded] = encoded.split('\t') - if (!ciphertextEncoded || !ivEncoded) throw new Error('encrypted request missing') - const ciphertext = Buffer.from(ciphertextEncoded, 'base64') - const iv = Buffer.from(ivEncoded, 'base64') - if (ciphertext.byteLength < 17 || iv.byteLength !== aesGcmIvBytes) { - throw new Error('encrypted request invalid') - } - const decipher = createDecipheriv('aes-256-gcm', key.bytes, iv) + const encrypted = await readEncryptedRequestParts( + input.command, + input.containerId, + input.fixture.jobId, + ) + const decipher = createDecipheriv('aes-256-gcm', key.bytes, encrypted.iv) decipher.setAAD(Buffer.from(input.fixture.requestContext, 'utf8')) - decipher.setAuthTag(ciphertext.subarray(-16)) + decipher.setAuthTag(encrypted.ciphertext.subarray(-16)) const plaintext = Buffer.concat([ - decipher.update(ciphertext.subarray(0, -16)), + decipher.update(encrypted.ciphertext.subarray(0, -16)), decipher.final(), ]).toString('utf8') - const request = JSON.parse(plaintext) + const request: unknown = JSON.parse(plaintext) deepStrictEqual(request, input.fixture.request) } catch { throw new Error('Alpha 4 persistence fixture encrypted request could not be decrypted.') } } + +export { seedAlpha4PersistenceFixture } +export type { + Alpha4PersistenceCommandInput, + Alpha4PersistenceFixture, + Alpha4PersistenceSnapshot, + AssertAlpha4PersistenceFixtureInput, + AssertAlpha4PersistenceSnapshotInput, + Command, + SeedAlpha4PersistenceFixtureInput, +} diff --git a/scripts/alpha4-persistence/crypto.ts b/scripts/alpha4-persistence/crypto.ts new file mode 100644 index 0000000..9d6b7db --- /dev/null +++ b/scripts/alpha4-persistence/crypto.ts @@ -0,0 +1,67 @@ +import { createCipheriv, createHash, randomBytes } from 'node:crypto' + +import type { Alpha4CertificateRequest } from './types' + +export const APP_ENCRYPTION_KEY_BYTES = 32 +export const AES_GCM_IV_BYTES = 12 + +function canonicalValue(value: unknown): unknown { + if (value instanceof Date) return value.toISOString() + if (Array.isArray(value)) return value.map(canonicalValue) + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value as Record) + .toSorted(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, canonicalValue(entry)]), + ) + } + return value +} + +export function digest(value: unknown): string { + return createHash('sha256') + .update(JSON.stringify(canonicalValue(value))) + .digest('hex') +} + +export function uuidV7(): string { + const bytes = randomBytes(16) + const timestamp = BigInt(Date.now()) + bytes[0] = Number((timestamp >> 40n) & 0xffn) + bytes[1] = Number((timestamp >> 32n) & 0xffn) + bytes[2] = Number((timestamp >> 24n) & 0xffn) + bytes[3] = Number((timestamp >> 16n) & 0xffn) + bytes[4] = Number((timestamp >> 8n) & 0xffn) + bytes[5] = Number(timestamp & 0xffn) + bytes[6] = (bytes[6]! & 0x0f) | 0x70 + bytes[8] = (bytes[8]! & 0x3f) | 0x80 + const hex = bytes.toString('hex') + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}` +} + +export function decodeApplicationKey(encoded: string): { + readonly bytes: Buffer + readonly digest: string +} { + const bytes = Buffer.from(encoded, 'base64') + if (bytes.byteLength !== APP_ENCRYPTION_KEY_BYTES || bytes.toString('base64') !== encoded) { + throw new Error('Alpha 4 fixture application encryption key is invalid.') + } + return { bytes, digest: createHash('sha256').update(bytes).digest('hex') } +} + +export function encryptRequest( + request: Alpha4CertificateRequest, + key: Buffer, + context: string, +): { readonly ciphertext: Buffer; readonly iv: Buffer } { + const iv = randomBytes(AES_GCM_IV_BYTES) + const cipher = createCipheriv('aes-256-gcm', key, iv) + cipher.setAAD(Buffer.from(context, 'utf8')) + const ciphertext = Buffer.concat([ + cipher.update(JSON.stringify(request), 'utf8'), + cipher.final(), + cipher.getAuthTag(), + ]) + return { ciphertext, iv } +} diff --git a/scripts/alpha4-persistence/seed-sql.ts b/scripts/alpha4-persistence/seed-sql.ts new file mode 100644 index 0000000..c7798da --- /dev/null +++ b/scripts/alpha4-persistence/seed-sql.ts @@ -0,0 +1,193 @@ +import type { Alpha4CertificateRequest, Alpha4PersistenceIds } from './types' +import { digest, encryptRequest, uuidV7 } from './crypto' +import { sqlJson, sqlQuote, sqlTextArray, sqlTimestamp } from './storage' + +const futureAttemptAt = '2099-01-01T00:00:00.000Z' +const futureLeaseExpiresAt = '2099-01-02T00:00:00.000Z' + +export function buildSeedStatements( + ids: Alpha4PersistenceIds, + input: { readonly runId: string }, + values: { + readonly key: Buffer + readonly now: Date + readonly operationStartedAt: Date + readonly lastSuccessAt: Date + readonly lastAttemptAt: Date + readonly certificateCreatedAt: Date + readonly jobCreatedAt: Date + readonly issuedAt: Date + readonly expiresAt: Date + readonly nextRenewalAt: Date + readonly domain: string + readonly identityDigest: string + readonly request: Alpha4CertificateRequest + readonly requestContext: string + readonly cursorStatement: string + }, +): string[] { + const host = { + id: ids.hostId, + forwardScheme: 'http', + forwardHost: '127.0.0.1', + forwardPort: 8080, + enabled: false, + certificateId: ids.certificateId, + forceHttps: false, + verifyUpstreamTls: true, + upstreamTlsServerName: null, + trustedCaId: null, + accessPolicyId: null, + createdAt: values.now, + updatedAt: values.now, + } + const hostRevision = digest({ host, domains: [values.domain], httpSettings: {} }) + const requestDigest = digest({ proxyHostId: ids.hostId, host: null, request: values.request }) + const operation = { + id: ids.operationId, + kind: 'renew', + stage: 'retry_scheduled', + startedAt: values.operationStartedAt.toISOString(), + updatedAt: values.now.toISOString(), + } + const encrypted = encryptRequest(values.request, values.key, values.requestContext) + const certificateFingerprint = `sha256:${digest({ runId: input.runId, certificate: true })}` + const leaseToken = uuidV7() + return [ + 'begin', + 'insert into rentnerproxy.users (id, display_name, email, status, created_at, updated_at) values (' + + sqlQuote(ids.ownerUserId) + + ', ' + + sqlQuote(`Alpha 4 Fixture Owner ${input.runId}`) + + ', ' + + sqlQuote(`alpha4-${values.identityDigest.slice(0, 24)}@fixture.invalid`) + + ", 'active', " + + sqlTimestamp(values.certificateCreatedAt) + + ', ' + + sqlTimestamp(values.now) + + ')', + 'insert into rentnerproxy.user_roles (user_id, role_id) select ' + + sqlQuote(ids.ownerUserId) + + ", id from rentnerproxy.roles where key = 'owner'", + 'insert into rentnerproxy.certificates (id, name, source, environment, status, operation, current_operation, challenge_type, issued_at, expires_at, issuer, fingerprint, candidate, dns_cleanup_pending, last_error_code, last_activated_at, last_error_at, next_attempt_at, attempt_count, last_attempt_at, last_success_at, next_renewal_at, controller_updated_at, created_at, updated_at) values (' + + sqlQuote(ids.certificateId) + + ', ' + + sqlQuote(`Alpha 4 persistence ${input.runId}`) + + ", 'acme', 'staging', 'valid', 'renewing', " + + sqlJson(operation) + + ", 'http-01', " + + sqlTimestamp(values.issuedAt) + + ', ' + + sqlTimestamp(values.expiresAt) + + ', ' + + sqlQuote('Alpha 4 Fixture CA') + + ', ' + + sqlQuote(certificateFingerprint) + + ', null, false, ' + + sqlQuote('acme_failed') + + ', ' + + sqlTimestamp(values.lastSuccessAt) + + ', ' + + sqlTimestamp(values.lastAttemptAt) + + ', ' + + sqlTimestamp(futureAttemptAt) + + ', 2, ' + + sqlTimestamp(values.lastAttemptAt) + + ', ' + + sqlTimestamp(values.lastSuccessAt) + + ', ' + + sqlTimestamp(values.nextRenewalAt) + + ', ' + + sqlTimestamp(values.now) + + ', ' + + sqlTimestamp(values.certificateCreatedAt) + + ', ' + + sqlTimestamp(values.now) + + ')', + 'insert into rentnerproxy.certificate_domains (certificate_id, domain, created_at) values (' + + sqlQuote(ids.certificateId) + + ', ' + + sqlQuote(values.domain) + + ', ' + + sqlTimestamp(values.certificateCreatedAt) + + ')', + 'insert into rentnerproxy.proxy_hosts (id, forward_scheme, forward_host, forward_port, enabled, certificate_id, force_https, verify_upstream_tls, upstream_tls_server_name, trusted_ca_id, access_policy_id, created_at, updated_at) values (' + + sqlQuote(ids.hostId) + + ", 'http', '127.0.0.1', 8080, false, " + + sqlQuote(ids.certificateId) + + ', false, true, null, null, null, ' + + sqlTimestamp(values.now) + + ', ' + + sqlTimestamp(values.now) + + ')', + 'insert into rentnerproxy.host_domains (proxy_host_id, domain, created_at) values (' + + sqlQuote(ids.hostId) + + ', ' + + sqlQuote(values.domain) + + ', ' + + sqlTimestamp(values.now) + + ')', + 'insert into rentnerproxy.certificate_binding_jobs (id, actor_user_id, proxy_host_id, certificate_id, idempotency_key, request_digest, domains, required_permissions, host_revision, assigned_revision, desired_enabled, desired_force_https, request_ciphertext, request_iv, stage, controller_stage, controller_operation_id, last_error_code, attempt_count, retry_requested, next_attempt_at, lease_token, lease_expires_at, created_at, updated_at) values (' + + sqlQuote(ids.jobId) + + ', ' + + sqlQuote(ids.ownerUserId) + + ', ' + + sqlQuote(ids.hostId) + + ', ' + + sqlQuote(ids.certificateId) + + ', ' + + sqlQuote(ids.idempotencyKey) + + ', ' + + sqlQuote(requestDigest) + + ', ' + + sqlTextArray([values.domain]) + + ', ' + + sqlTextArray(['proxy_hosts.update', 'certificates.issue']) + + ', ' + + sqlQuote(hostRevision) + + ', ' + + sqlQuote(hostRevision) + + ', false, false, decode(' + + sqlQuote(encrypted.ciphertext.toString('base64')) + + ", 'base64'), decode(" + + sqlQuote(encrypted.iv.toString('base64')) + + ", 'base64'), 'issuing', 'retry_scheduled', " + + sqlQuote(ids.operationId) + + ", 'acme_failed', 2, true, " + + sqlTimestamp(futureAttemptAt) + + ', ' + + sqlQuote(leaseToken) + + ', ' + + sqlTimestamp(futureLeaseExpiresAt) + + ', ' + + sqlTimestamp(values.jobCreatedAt) + + ', ' + + sqlTimestamp(values.now) + + ')', + 'insert into rentnerproxy.certificate_event_receipts (event_id, operation_id, certificate_id, kind, stage, occurred_at, error_code, received_at) values (' + + sqlQuote(ids.eventIds[0]) + + ', ' + + sqlQuote(ids.operationId) + + ', ' + + sqlQuote(ids.certificateId) + + ", 'started', 'creating_order', " + + sqlTimestamp(values.operationStartedAt) + + ', null, ' + + sqlTimestamp(values.now) + + '), (' + + sqlQuote(ids.eventIds[1]) + + ', ' + + sqlQuote(ids.operationId) + + ', ' + + sqlQuote(ids.certificateId) + + ", 'retry_scheduled', 'retry_scheduled', " + + sqlTimestamp(values.lastAttemptAt) + + ', ' + + sqlQuote('acme_failed') + + ', ' + + sqlTimestamp(values.now) + + ')', + values.cursorStatement, + 'commit', + ] +} diff --git a/scripts/alpha4-persistence/seed.ts b/scripts/alpha4-persistence/seed.ts new file mode 100644 index 0000000..fca9cae --- /dev/null +++ b/scripts/alpha4-persistence/seed.ts @@ -0,0 +1,107 @@ +import type { + Alpha4CertificateRequest, + Alpha4PersistenceFixture, + Alpha4PersistenceIds, + Command, +} from './types' +import { decodeApplicationKey, digest, uuidV7 } from './crypto' +import { buildSeedStatements } from './seed-sql' +import { psql, readContainerFile, readOrCreateCursor } from './storage' + +const appEncryptionKeyFile = '/run/rentnerproxy/app-key/value' + +function validateRunId(runId: string): void { + if (!/^[a-z0-9][a-z0-9_-]{5,63}$/u.test(runId)) { + throw new Error('Alpha 4 fixture runId must be a bounded lowercase identifier.') + } +} + +function validateCommandInput(containerId: string): void { + if (!containerId.trim() || containerId.length > 256) { + throw new Error('Alpha 4 fixture container id is invalid.') + } +} + +function readFixtureIds(): Alpha4PersistenceIds { + return { + ownerUserId: uuidV7(), + hostId: uuidV7(), + certificateId: uuidV7(), + jobId: uuidV7(), + idempotencyKey: uuidV7(), + operationId: uuidV7(), + eventIds: [uuidV7(), uuidV7()], + } +} + +export async function seedAlpha4PersistenceFixture(input: { + readonly command: Command + readonly containerId: string + readonly runId: string +}): Promise { + validateCommandInput(input.containerId) + validateRunId(input.runId) + const encodedKey = ( + await readContainerFile(input.command, input.containerId, appEncryptionKeyFile) + ).trim() + const key = decodeApplicationKey(encodedKey) + const ids = readFixtureIds() + const now = new Date() + const operationStartedAt = new Date(now.getTime() - 10 * 60_000) + const lastSuccessAt = new Date(now.getTime() - 24 * 60 * 60_000) + const lastAttemptAt = new Date(now.getTime() - 5 * 60_000) + const certificateCreatedAt = new Date(now.getTime() - 20 * 60_000) + const jobCreatedAt = new Date(now.getTime() - 15 * 60_000) + const issuedAt = new Date(now.getTime() - 30 * 24 * 60 * 60_000) + const expiresAt = new Date(now.getTime() + 60 * 24 * 60 * 60_000) + const nextRenewalAt = new Date(now.getTime() + 30 * 24 * 60 * 60_000) + const identityDigest = digest({ runId: input.runId, ownerUserId: ids.ownerUserId }) + const domain = `a4-${identityDigest.slice(0, 32)}.example.com` + const request: Alpha4CertificateRequest = { + name: `Alpha 4 persistence ${input.runId}`, + domains: [domain], + environment: 'staging', + challengeType: 'http-01', + contactEmail: '', + acceptTerms: true, + } + const requestContext = `certificate-binding-job:${ids.jobId}` + const cursor = await readOrCreateCursor(input.command, input.containerId, now) + const statements = buildSeedStatements( + ids, + { runId: input.runId }, + { + key: key.bytes, + now, + operationStartedAt, + lastSuccessAt, + lastAttemptAt, + certificateCreatedAt, + jobCreatedAt, + issuedAt, + expiresAt, + nextRenewalAt, + domain, + identityDigest, + request, + requestContext, + cursorStatement: cursor.statement, + }, + ) + await psql(input.command, input.containerId, statements.join(';\n') + ';', 60_000) + return { + runId: input.runId, + ownerUserId: ids.ownerUserId, + hostId: ids.hostId, + certificateId: ids.certificateId, + jobId: ids.jobId, + idempotencyKey: ids.idempotencyKey, + operationId: ids.operationId, + eventIds: ids.eventIds, + domain, + cursor: cursor.cursor, + applicationKeyDigest: key.digest, + requestContext, + request, + } +} diff --git a/scripts/alpha4-persistence/storage.ts b/scripts/alpha4-persistence/storage.ts new file mode 100644 index 0000000..9f00a5e --- /dev/null +++ b/scripts/alpha4-persistence/storage.ts @@ -0,0 +1,163 @@ +import type { Alpha4PersistenceFixture, Alpha4PersistenceSnapshot, Command } from './types' +import { uuidV7 } from './crypto' + +const postgresPasswordFile = '/run/rentnerproxy/postgres/value' +const databaseName = 'rentnerproxy' +const databaseUser = 'rentnerproxy' + +function shellQuote(value: string): string { + return "'" + value.replaceAll("'", "'\"'\"'") + "'" +} + +export function sqlQuote(value: string): string { + return "'" + value.replaceAll("'", "''") + "'" +} + +export function sqlTimestamp(value: Date | string): string { + const serialized = value instanceof Date ? value.toISOString() : value + return `${sqlQuote(serialized)}::timestamptz` +} + +export function sqlJson(value: unknown): string { + return `${sqlQuote(JSON.stringify(value))}::jsonb` +} + +export function sqlTextArray(values: readonly string[]): string { + return `ARRAY[${values.map(sqlQuote).join(',')}]::text[]` +} + +export async function inContainer( + command: Command, + containerId: string, + script: string, + timeoutMs = 60_000, +): Promise { + return command(['docker', 'exec', containerId, 'sh', '-ceu', script], timeoutMs) +} + +export async function readContainerFile( + command: Command, + containerId: string, + path: string, +): Promise { + return inContainer(command, containerId, `cat ${path}`) +} + +export async function psql( + command: Command, + containerId: string, + statement: string, + timeoutMs = 60_000, +): Promise { + const script = + 'PGPASSWORD="$(cat ' + + postgresPasswordFile + + ')" gosu postgres psql --no-psqlrc --no-password --quiet --set=ON_ERROR_STOP=1 ' + + '--tuples-only --no-align --host=127.0.0.1 --username=' + + databaseUser + + ' --dbname=' + + databaseName + + ' --command=' + + shellQuote(statement) + return inContainer(command, containerId, script, timeoutMs) +} + +function isCursor(value: string): boolean { + return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}:\d+$/iu.test(value) +} + +export async function readOrCreateCursor( + command: Command, + containerId: string, + now: Date, +): Promise<{ readonly cursor: string; readonly statement: string }> { + const existing = ( + await psql( + command, + containerId, + "select coalesce(cursor, '') from rentnerproxy.certificate_event_cursor where id = 1", + ) + ).trim() + if (isCursor(existing)) { + return { + cursor: existing, + statement: + 'insert into rentnerproxy.certificate_event_cursor (id, cursor, updated_at) values (1, ' + + sqlQuote(existing) + + ', ' + + sqlTimestamp(now) + + ') on conflict (id) do nothing', + } + } + const cursor = `${uuidV7()}:0` + return { + cursor, + statement: + 'insert into rentnerproxy.certificate_event_cursor (id, cursor, updated_at) values (1, ' + + sqlQuote(cursor) + + ', ' + + sqlTimestamp(now) + + ') on conflict (id) do update set cursor = excluded.cursor, updated_at = excluded.updated_at', + } +} + +export function snapshotStatement(fixture: Alpha4PersistenceFixture): string { + return `select jsonb_build_object( + 'fixtureVersion', 1, + 'user', (select to_jsonb(u) - 'password_hash' from rentnerproxy.users u where u.id = ${sqlQuote(fixture.ownerUserId)}), + 'userRoles', (select coalesce(jsonb_agg(to_jsonb(ur) order by ur.role_id), '[]'::jsonb) from rentnerproxy.user_roles ur where ur.user_id = ${sqlQuote(fixture.ownerUserId)}), + 'certificate', (select to_jsonb(c) from rentnerproxy.certificates c where c.id = ${sqlQuote(fixture.certificateId)}), + 'certificateDomains', (select coalesce(jsonb_agg(to_jsonb(d) order by d.domain, d.id), '[]'::jsonb) from rentnerproxy.certificate_domains d where d.certificate_id = ${sqlQuote(fixture.certificateId)}), + 'proxyHost', (select to_jsonb(h) from rentnerproxy.proxy_hosts h where h.id = ${sqlQuote(fixture.hostId)}), + 'hostDomains', (select coalesce(jsonb_agg(to_jsonb(d) order by d.domain, d.id), '[]'::jsonb) from rentnerproxy.host_domains d where d.proxy_host_id = ${sqlQuote(fixture.hostId)}), + 'job', (select (to_jsonb(j) - 'request_ciphertext' - 'request_iv') || jsonb_build_object('request_ciphertext_bytes', octet_length(j.request_ciphertext), 'request_ciphertext_md5', md5(encode(j.request_ciphertext, 'hex')), 'request_iv_bytes', octet_length(j.request_iv), 'request_iv_md5', md5(encode(j.request_iv, 'hex'))) from rentnerproxy.certificate_binding_jobs j where j.id = ${sqlQuote(fixture.jobId)}), + 'eventReceipts', (select coalesce(jsonb_agg(to_jsonb(r) order by r.event_id), '[]'::jsonb) from rentnerproxy.certificate_event_receipts r where r.event_id in (${fixture.eventIds.map(sqlQuote).join(',')})), + 'eventCursor', (select to_jsonb(c) from rentnerproxy.certificate_event_cursor c where c.id = 1) + )::text` +} + +function parseJsonRecord(value: string, label: string): Alpha4PersistenceSnapshot { + try { + const parsed: unknown = JSON.parse(value) + if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { + throw new Error(`Alpha 4 fixture returned invalid ${label}.`) + } + return parsed as Alpha4PersistenceSnapshot + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`Alpha 4 fixture returned invalid ${label}.`, { cause: error }) + } + throw error + } +} + +export async function readAlpha4PersistenceSnapshot( + command: Command, + containerId: string, + fixture: Alpha4PersistenceFixture, +): Promise { + const output = await psql(command, containerId, snapshotStatement(fixture)) + return parseJsonRecord(output, 'persistence snapshot') +} + +export async function readEncryptedRequestParts( + command: Command, + containerId: string, + jobId: string, +): Promise<{ readonly ciphertext: Buffer; readonly iv: Buffer }> { + const encoded = ( + await psql( + command, + containerId, + `select encode(request_ciphertext, 'base64') || E'\\t' || encode(request_iv, 'base64') from rentnerproxy.certificate_binding_jobs where id = ${sqlQuote(jobId)}`, + ) + ).trim() + const [ciphertextEncoded, ivEncoded] = encoded.split('\t') + if (!ciphertextEncoded || !ivEncoded) throw new Error('encrypted request missing') + const ciphertext = Buffer.from(ciphertextEncoded, 'base64') + const iv = Buffer.from(ivEncoded, 'base64') + if (ciphertext.byteLength < 17 || iv.byteLength !== 12) { + throw new Error('encrypted request invalid') + } + return { ciphertext, iv } +} diff --git a/scripts/alpha4-persistence/types.ts b/scripts/alpha4-persistence/types.ts new file mode 100644 index 0000000..b17defa --- /dev/null +++ b/scripts/alpha4-persistence/types.ts @@ -0,0 +1,55 @@ +export type Command = (args: string[], timeoutMs?: number) => Promise + +export interface Alpha4CertificateRequest { + readonly name: string + readonly domains: readonly string[] + readonly environment: 'staging' + readonly challengeType: 'http-01' + readonly contactEmail: '' + readonly acceptTerms: true +} + +export interface Alpha4PersistenceFixture { + readonly runId: string + readonly ownerUserId: string + readonly hostId: string + readonly certificateId: string + readonly jobId: string + readonly idempotencyKey: string + readonly operationId: string + readonly eventIds: readonly string[] + readonly domain: string + readonly cursor: string + readonly applicationKeyDigest: string + readonly requestContext: string + readonly request: Alpha4CertificateRequest +} + +export interface Alpha4PersistenceIds { + readonly ownerUserId: string + readonly hostId: string + readonly certificateId: string + readonly jobId: string + readonly idempotencyKey: string + readonly operationId: string + readonly eventIds: readonly [string, string] +} + +export type Alpha4PersistenceSnapshot = Readonly> + +export interface Alpha4PersistenceCommandInput { + readonly command: Command + readonly containerId: string +} + +export interface SeedAlpha4PersistenceFixtureInput extends Alpha4PersistenceCommandInput { + readonly runId: string +} + +export interface AssertAlpha4PersistenceFixtureInput extends Alpha4PersistenceCommandInput { + readonly fixture: Alpha4PersistenceFixture +} + +export interface AssertAlpha4PersistenceSnapshotInput extends AssertAlpha4PersistenceFixtureInput { + readonly expected: Alpha4PersistenceSnapshot +} From 51361de43be607ef45bee21d3f182fa9088c928b Mon Sep 17 00:00:00 2001 From: "HomePC\\Kevin" Date: Sun, 13 Sep 2026 00:44:27 +0200 Subject: [PATCH 5/6] test(release): verify durable jobs and production restore boundaries --- README.md | 18 ++++++++ scripts/appliance-compose-smoke.ts | 54 +++++++++++++++++++++- scripts/certificate-smoke.ts | 7 +-- scripts/certificate-state-restore-smoke.ts | 17 ++++++- scripts/controller-state-archive.ts | 1 + scripts/production-restore.ts | 26 +---------- 6 files changed, 90 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 87354b9..ae56bda 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,24 @@ bun --env-file=/srv/rentnerproxy/.env scripts/production-restore.ts \ Restore replaces the target project's data. Stop the original appliance before recovery uses the same host ports, and preserve its volumes until recovery health and traffic are verified. +### Alpha 4 upgrade verification + +The production smoke gate upgrades the published, digest-pinned Alpha 3 appliance in place, +checks migrations and existing users, roles, hosts, certificate fingerprints and traffic, +then verifies repeated starts, backup/restore into fresh volumes and failed-restore rollback. +The earlier Alpha 1 upgrade path remains covered. + +Certificate recovery tests archive real active and candidate material, ACME accounts, +operation journals, retry state and encrypted DNS credentials using the production archive +filters. They restore the files before restarting the controller with the local CA offline. +The production backup/restore test also compares durable binding jobs, renewal metadata, +event receipts and cursor state, and verifies request decryption with the restored application key. + +Run `bun run check`, `bun run certificates:smoke` and `bun run production:smoke` for this gate. +The required Production Smokes CI job also checks proxy forwarding and upstream TLS. +Keep the deployment environment, including `RENTNERPROXY_PUBLIC_ORIGIN`, trusted proxy CIDRs +and TCP/UDP port mappings, alongside the backup: these settings are not stored in PostgreSQL. + ## Features - Caddy 2.11.4 proxy hosts for HTTP, HTTPS, redirects, and WebSockets. diff --git a/scripts/appliance-compose-smoke.ts b/scripts/appliance-compose-smoke.ts index 0abd496..4381b51 100644 --- a/scripts/appliance-compose-smoke.ts +++ b/scripts/appliance-compose-smoke.ts @@ -12,6 +12,12 @@ import { fileURLToPath } from 'node:url' import { restoreSmokeDiagnostic, smokeCompose, smokeDockerArguments } from './smoke-resources' import { buildHttp3Client, requestHttp3Client, assertHttp3Response } from './http3-client' import { verifyAlpha1Upgrade, verifyAlpha3Upgrade } from './alpha1-upgrade-smoke' +import { + seedAlpha4PersistenceFixture, + readAlpha4PersistenceSnapshot, + assertAlpha4PersistenceFixture, + assertAlpha4PersistenceRequestDecrypts, +} from './alpha4-persistence-fixture' const repositoryRoot = fileURLToPath(new URL('..', import.meta.url)) const rootComposeFile = join(repositoryRoot, 'docker-compose.yml') @@ -19,6 +25,7 @@ const productionDockerfile = join(repositoryRoot, 'docker', 'production', 'Docke const runId = randomUUID().replaceAll('-', '').slice(0, 12) const project = 'rentnerproxy-appliance-smoke-' + runId const publicOrigin = 'https://management.appliance-smoke.invalid' +const trustedProxyCidrs = '127.0.0.1/32,::1/128' const smtpEnvironment = { SMTP_FROM: 'RentnerProxy ', SMTP_HOST: 'smtp.appliance-smoke.invalid', @@ -268,6 +275,8 @@ async function runSmoke(): Promise { .join('\n') + '\nRENTNERPROXY_PUBLIC_ORIGIN=' + publicOrigin + + '\nRENTNERPROXY_PROXY_TRUSTED_PROXY_CIDRS=' + + trustedProxyCidrs + '\n', 'utf8', ) @@ -350,7 +359,7 @@ async function runSmoke(): Promise { 'RENTNERPROXY_PUBLIC_ORIGIN', ].toSorted(), ) - assert.equal(service.environment?.RENTNERPROXY_PROXY_TRUSTED_PROXY_CIDRS, '') + assert.equal(service.environment?.RENTNERPROXY_PROXY_TRUSTED_PROXY_CIDRS, trustedProxyCidrs) assert.equal(service.environment?.RENTNERPROXY_PUBLIC_ORIGIN, publicOrigin) assert.deepEqual( (service.ports ?? []).map(({ published, target, protocol }) => ({ @@ -661,6 +670,9 @@ async function runSmoke(): Promise { passed('database, Redis, and controller are unpublished and loopback-only') const environment = JSON.parse(await inspect(id, '{{json .Config.Env}}')) as string[] + assert.ok( + environment.includes('RENTNERPROXY_PROXY_TRUSTED_PROXY_CIDRS=' + trustedProxyCidrs), + ) for (const name of smtpNames) { assert.ok( environment.includes( @@ -945,6 +957,24 @@ async function runSmoke(): Promise { 'sha256sum', proxyBackupMarker, ]) + const persistenceFixture = await seedAlpha4PersistenceFixture({ + command, + containerId: recreatedId, + runId, + }) + const persistenceSnapshot = await readAlpha4PersistenceSnapshot({ + command, + containerId: recreatedId, + fixture: persistenceFixture, + }) + await assertAlpha4PersistenceRequestDecrypts({ + command, + containerId: recreatedId, + fixture: persistenceFixture, + }) + passed( + 'durable binding jobs, renewal retry metadata and event receipts are present before backup', + ) const backupRoot = join(temporaryRoot, 'backups') await commandWithEnvironment( [ @@ -1022,6 +1052,28 @@ async function runSmoke(): Promise { ) const restoredId = await containerId(restoreCompose) await waitForHealthy(restoredId) + const restoredEnvironment = JSON.parse( + await inspect(restoredId, '{{json .Config.Env}}'), + ) as string[] + assert.ok( + restoredEnvironment.includes( + 'RENTNERPROXY_PROXY_TRUSTED_PROXY_CIDRS=' + trustedProxyCidrs, + ), + ) + await assertAlpha4PersistenceFixture({ + command, + containerId: restoredId, + fixture: persistenceFixture, + expected: persistenceSnapshot, + }) + await assertAlpha4PersistenceRequestDecrypts({ + command, + containerId: restoredId, + fixture: persistenceFixture, + }) + passed( + 'backup restores exact binding jobs, retry states and event journal data with decryptable requests', + ) const disasterRestoreSecrets = JSON.parse( await command([ 'docker', diff --git a/scripts/certificate-smoke.ts b/scripts/certificate-smoke.ts index b640fe6..89a7a01 100644 --- a/scripts/certificate-smoke.ts +++ b/scripts/certificate-smoke.ts @@ -2000,8 +2000,6 @@ async function runSmoke(): Promise { 'certificates/' + acmeId + '/candidate.json', ], }) - await refreshRuntimeDns() - await waitForRuntimeReady() passed( 'production archive restores active material, issued candidate, operation journal and ACME account', ) @@ -2232,8 +2230,6 @@ async function runSmoke(): Promise { const cleanupRecoveredFingerprint = cleanupFailure.fingerprint assert.ok(dnsFixture.records.length > 0) await checkWildcardTraffic() - dnsFixture.failCleanup = false - await command(['docker', 'network', 'disconnect', network, pebbleContainer], { timeoutMs: 30_000, }) @@ -2247,9 +2243,8 @@ async function runSmoke(): Promise { 'certificates/acme-accounts/staging.json', ], }) - await refreshRuntimeDns() - await waitForRuntimeReady() passed('production archive restores encrypted DNS credentials and pending cleanup state') + dnsFixture.failCleanup = false await restartRuntime() await waitFor( async () => (await controllerRequest('/internal/v1/proxy/status')).status === 200, diff --git a/scripts/certificate-state-restore-smoke.ts b/scripts/certificate-state-restore-smoke.ts index ee87ece..855dd1c 100644 --- a/scripts/certificate-state-restore-smoke.ts +++ b/scripts/certificate-state-restore-smoke.ts @@ -14,11 +14,27 @@ export async function restoreCertificateStateFixture( const archiveVolume = input.volume + '-restore-' + randomUUID().replaceAll('-', '') await command(['docker', 'volume', 'create', archiveVolume]) try { + await command([ + 'docker', + 'run', + '--rm', + '--user', + '0:0', + '--entrypoint', + 'chown', + '--volume', + archiveVolume + ':/archive', + input.image, + '10001:10001', + '/archive', + ]) await command(['docker', 'stop', input.container], { timeoutMs: 60_000 }) const run = [ 'docker', 'run', '--rm', + '--user', + '10001:10001', '--entrypoint', 'sh', '--volume', @@ -54,7 +70,6 @@ export async function restoreCertificateStateFixture( 'cd /state; find . -type f -exec sha256sum {} + | LC_ALL=C sort > /archive/restored.sha256; ' + 'cmp /archive/expected.sha256 /archive/restored.sha256', ]) - await command(['docker', 'start', input.container]) } finally { await command(['docker', 'volume', 'rm', archiveVolume]).catch(() => undefined) } diff --git a/scripts/controller-state-archive.ts b/scripts/controller-state-archive.ts index 60346e8..8c18b6f 100644 --- a/scripts/controller-state-archive.ts +++ b/scripts/controller-state-archive.ts @@ -4,6 +4,7 @@ export const stateArchiveExclusions = [ './last-known-good.conf', './last-good.conf', './engine.pid', + './*.pid', './host-configs', './host-configs/**', './caddy-admin.sock', diff --git a/scripts/production-restore.ts b/scripts/production-restore.ts index 55a8aff..7f39f8e 100644 --- a/scripts/production-restore.ts +++ b/scripts/production-restore.ts @@ -15,31 +15,7 @@ const databaseUser = 'rentnerproxy' const stateArchiveName = 'controller-state.tar' const bootstrapScript = '/opt/rentnerproxy/web/docker/web/bootstrap-secrets.mjs' const healthcheckScript = '/opt/rentnerproxy/web/docker/web/healthcheck.mjs' -const legacyRuntimeStateExclusions = [ - './active.conf', - './candidate.conf', - './last-known-good.conf', - './last-good.conf', - './engine.pid', - './host-configs', - './host-configs/**', - './*.pid', - './runtime-probe.sock', - './caddy-admin.sock', - './caddy/**/*.sock', - './caddy/**/*.tmp', - './caddy/**/*.lock', - './cache', - './bootstrap', - './runtime', - './tmp', - './run', - './log', - './logs', - '*.log', - '*.log.*', - '.*.tmp', -] +import { stateArchiveExclusions as legacyRuntimeStateExclusions } from './controller-state-archive' function optionValue(argumentsList: string[], name: string): string | undefined { const index = argumentsList.indexOf(name) From d40bdc877278f2fa708477f78c1313c576ae3d6d Mon Sep 17 00:00:00 2001 From: "HomePC\\Kevin" Date: Sun, 13 Sep 2026 00:57:44 +0200 Subject: [PATCH 6/6] test(release): verify restored HTTP3 and deployment configuration --- README.md | 2 +- scripts/alpha1-upgrade-smoke.ts | 5 ++--- scripts/appliance-compose-smoke.ts | 6 ++++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ae56bda..0ab35d1 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ management origin and an SMTP host, user, password, and sender address are requi Download [`docker-compose.yml`](./docker-compose.yml) and [`.env.production.example`](./.env.production.example) to an empty folder. Set the Compose image to -`ghcr.io/rentnerkev/rentnerproxy:v1.0.0-alpha.2`, copy the environment template, and set +`ghcr.io/rentnerkev/rentnerproxy:v1.0.0-alpha.3`, copy the environment template, and set `RENTNERPROXY_PUBLIC_ORIGIN` and the SMTP values: ```bash diff --git a/scripts/alpha1-upgrade-smoke.ts b/scripts/alpha1-upgrade-smoke.ts index 71e9cc2..ea0132b 100644 --- a/scripts/alpha1-upgrade-smoke.ts +++ b/scripts/alpha1-upgrade-smoke.ts @@ -29,7 +29,7 @@ export interface PublishedUpgradeBaseline { readonly version: string readonly revision: string readonly migrationCount: number - readonly targetName: 'Alpha 2' | 'Alpha 4' + readonly targetName: 'Alpha 4' readonly directoryName: string } @@ -39,7 +39,7 @@ export const ALPHA1_BASELINE: PublishedUpgradeBaseline = { version: 'v1.0.0-alpha.1', revision: ALPHA1_REVISION, migrationCount: ALPHA1_MIGRATION_COUNT, - targetName: 'Alpha 2', + targetName: 'Alpha 4', directoryName: 'alpha1-upgrade', } @@ -415,4 +415,3 @@ export async function verifyAlpha1Upgrade(options: UpgradeSmokeOptions): Promise export async function verifyAlpha3Upgrade(options: UpgradeSmokeOptions): Promise { return verifyPublishedUpgrade(options, ALPHA3_BASELINE) } - diff --git a/scripts/appliance-compose-smoke.ts b/scripts/appliance-compose-smoke.ts index 4381b51..bdb20be 100644 --- a/scripts/appliance-compose-smoke.ts +++ b/scripts/appliance-compose-smoke.ts @@ -325,6 +325,7 @@ async function runSmoke(): Promise { ...commandEnvironment, ...smtpEnvironment, RENTNERPROXY_PUBLIC_ORIGIN: publicOrigin, + RENTNERPROXY_PROXY_TRUSTED_PROXY_CIDRS: trustedProxyCidrs, RENTNERPROXY_COMPOSE_FILE: temporaryComposeFile, } @@ -1055,6 +1056,7 @@ async function runSmoke(): Promise { const restoredEnvironment = JSON.parse( await inspect(restoredId, '{{json .Config.Env}}'), ) as string[] + assert.ok(restoredEnvironment.includes('RENTNERPROXY_PUBLIC_ORIGIN=' + publicOrigin)) assert.ok( restoredEnvironment.includes( 'RENTNERPROXY_PROXY_TRUSTED_PROXY_CIDRS=' + trustedProxyCidrs, @@ -1115,6 +1117,10 @@ async function runSmoke(): Promise { restoredId, 'v3 restore heals Caddy from desired DB and preserves HTTP/HTTPS traffic', ) + await assertPublishedQuic() + passed( + 'verified HTTP/3 and deployment origin survive restore with trusted proxy configuration', + ) await command([ 'docker', 'exec',